From a2b0c9778feac970524b98ef7a91b5528fdeb9d5 Mon Sep 17 00:00:00 2001 From: Andy Goldstein Date: Fri, 27 Feb 2015 02:23:50 +0000 Subject: [PATCH] Add ability to refer to image by name + digest Add ability to refer to an image by repository name and digest using the format repository@digest. Works for pull, push, run, build, and rmi. Signed-off-by: Andy Goldstein --- Dockerfile | 2 +- api/client/commands.go | 42 +- daemon/image_delete.go | 5 +- daemon/list.go | 3 +- docs/man/docker-images.1.md | 4 + .../reference/api/docker_remote_api.md | 4 + .../reference/api/docker_remote_api_v1.18.md | 39 ++ docs/sources/reference/builder.md | 9 +- docs/sources/reference/commandline/cli.md | 42 +- docs/sources/reference/run.md | 10 +- graph/history.go | 3 +- graph/import.go | 2 +- graph/list.go | 28 +- graph/pull.go | 35 +- graph/push.go | 20 +- graph/tags.go | 139 +++-- graph/tags_unit_test.go | 40 ++ integration-cli/docker_cli_by_digest_test.go | 535 ++++++++++++++++++ integration-cli/docker_cli_push_test.go | 2 +- pkg/parsers/parsers.go | 10 +- pkg/parsers/parsers_test.go | 9 + registry/session_v2.go | 40 +- registry/v2/regexp.go | 3 + registry/v2/routes.go | 8 +- registry/v2/routes_test.go | 12 +- registry/v2/urls.go | 6 +- utils/utils.go | 17 + utils/utils_test.go | 30 + 28 files changed, 984 insertions(+), 115 deletions(-) create mode 100644 integration-cli/docker_cli_by_digest_test.go diff --git a/Dockerfile b/Dockerfile index 07c30c93610f4..080cc106a7398 100644 --- a/Dockerfile +++ b/Dockerfile @@ -108,7 +108,7 @@ RUN go get golang.org/x/tools/cmd/cover RUN gem install --no-rdoc --no-ri fpm --version 1.3.2 # Install registry -ENV REGISTRY_COMMIT c448e0416925a9876d5576e412703c9b8b865e19 +ENV REGISTRY_COMMIT b4cc5e3ecc2e9f4fa0e95d94c389e1d79e902486 RUN set -x \ && git clone https://github.com/docker/distribution.git /go/src/github.com/docker/distribution \ && (cd /go/src/github.com/docker/distribution && git checkout -q $REGISTRY_COMMIT) \ diff --git a/api/client/commands.go b/api/client/commands.go index 4b7157b80d8e7..1c03cdaec6600 100644 --- a/api/client/commands.go +++ b/api/client/commands.go @@ -1312,7 +1312,7 @@ func (cli *DockerCli) CmdPush(args ...string) error { } func (cli *DockerCli) CmdPull(args ...string) error { - cmd := cli.Subcmd("pull", "NAME[:TAG]", "Pull an image or a repository from the registry", true) + cmd := cli.Subcmd("pull", "NAME[:TAG|@DIGEST]", "Pull an image or a repository from the registry", true) allTags := cmd.Bool([]string{"a", "-all-tags"}, false, "Download all tagged images in the repository") cmd.Require(flag.Exact, 1) @@ -1325,7 +1325,7 @@ func (cli *DockerCli) CmdPull(args ...string) error { ) taglessRemote, tag := parsers.ParseRepositoryTag(remote) if tag == "" && !*allTags { - newRemote = taglessRemote + ":" + graph.DEFAULTTAG + newRemote = utils.ImageReference(taglessRemote, graph.DEFAULTTAG) } if tag != "" && *allTags { return fmt.Errorf("tag can't be used with --all-tags/-a") @@ -1378,6 +1378,7 @@ func (cli *DockerCli) CmdImages(args ...string) error { quiet := cmd.Bool([]string{"q", "-quiet"}, false, "Only show numeric IDs") all := cmd.Bool([]string{"a", "-all"}, false, "Show all images (default hides intermediate images)") noTrunc := cmd.Bool([]string{"#notrunc", "-no-trunc"}, false, "Don't truncate output") + showDigests := cmd.Bool([]string{"-digests"}, false, "Show digests") // FIXME: --viz and --tree are deprecated. Remove them in a future version. flViz := cmd.Bool([]string{"#v", "#viz", "#-viz"}, false, "Output graph in graphviz format") flTree := cmd.Bool([]string{"#t", "#tree", "#-tree"}, false, "Output graph in tree format") @@ -1504,20 +1505,43 @@ func (cli *DockerCli) CmdImages(args ...string) error { w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) if !*quiet { - fmt.Fprintln(w, "REPOSITORY\tTAG\tIMAGE ID\tCREATED\tVIRTUAL SIZE") + if *showDigests { + fmt.Fprintln(w, "REPOSITORY\tTAG\tDIGEST\tIMAGE ID\tCREATED\tVIRTUAL SIZE") + } else { + fmt.Fprintln(w, "REPOSITORY\tTAG\tIMAGE ID\tCREATED\tVIRTUAL SIZE") + } } for _, out := range outs.Data { - for _, repotag := range out.GetList("RepoTags") { + outID := out.Get("Id") + if !*noTrunc { + outID = common.TruncateID(outID) + } + // Tags referring to this image ID. + for _, repotag := range out.GetList("RepoTags") { repo, tag := parsers.ParseRepositoryTag(repotag) - outID := out.Get("Id") - if !*noTrunc { - outID = common.TruncateID(outID) + + if !*quiet { + if *showDigests { + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", repo, tag, "", outID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(out.GetInt64("Created"), 0))), units.HumanSize(float64(out.GetInt64("VirtualSize")))) + } else { + fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t%s\n", repo, tag, outID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(out.GetInt64("Created"), 0))), units.HumanSize(float64(out.GetInt64("VirtualSize")))) + } + } else { + fmt.Fprintln(w, outID) } + } + // Digests referring to this image ID. + for _, repoDigest := range out.GetList("RepoDigests") { + repo, digest := parsers.ParseRepositoryTag(repoDigest) if !*quiet { - fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t%s\n", repo, tag, outID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(out.GetInt64("Created"), 0))), units.HumanSize(float64(out.GetInt64("VirtualSize")))) + if *showDigests { + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", repo, "", digest, outID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(out.GetInt64("Created"), 0))), units.HumanSize(float64(out.GetInt64("VirtualSize")))) + } else { + fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t%s\n", repo, "", outID, units.HumanDuration(time.Now().UTC().Sub(time.Unix(out.GetInt64("Created"), 0))), units.HumanSize(float64(out.GetInt64("VirtualSize")))) + } } else { fmt.Fprintln(w, outID) } @@ -2208,7 +2232,7 @@ func (cli *DockerCli) createContainer(config *runconfig.Config, hostConfig *runc if tag == "" { tag = graph.DEFAULTTAG } - fmt.Fprintf(cli.err, "Unable to find image '%s:%s' locally\n", repo, tag) + fmt.Fprintf(cli.err, "Unable to find image '%s' locally\n", utils.ImageReference(repo, tag)) // we don't want to write to stdout anything apart from container.ID if err = cli.pullImageCustomOut(config.Image, cli.err); err != nil { diff --git a/daemon/image_delete.go b/daemon/image_delete.go index c193164765dfd..0c0a534cfdaf8 100644 --- a/daemon/image_delete.go +++ b/daemon/image_delete.go @@ -9,6 +9,7 @@ import ( "github.com/docker/docker/image" "github.com/docker/docker/pkg/common" "github.com/docker/docker/pkg/parsers" + "github.com/docker/docker/utils" ) func (daemon *Daemon) ImageDelete(job *engine.Job) engine.Status { @@ -48,7 +49,7 @@ func (daemon *Daemon) DeleteImage(eng *engine.Engine, name string, imgs *engine. img, err := daemon.Repositories().LookupImage(name) if err != nil { if r, _ := daemon.Repositories().Get(repoName); r != nil { - return fmt.Errorf("No such image: %s:%s", repoName, tag) + return fmt.Errorf("No such image: %s", utils.ImageReference(repoName, tag)) } return fmt.Errorf("No such image: %s", name) } @@ -102,7 +103,7 @@ func (daemon *Daemon) DeleteImage(eng *engine.Engine, name string, imgs *engine. } if tagDeleted { out := &engine.Env{} - out.Set("Untagged", repoName+":"+tag) + out.Set("Untagged", utils.ImageReference(repoName, tag)) imgs.Add(out) eng.Job("log", "untag", img.ID, "").Run() } diff --git a/daemon/list.go b/daemon/list.go index 174ec7ec75923..130ac05376c60 100644 --- a/daemon/list.go +++ b/daemon/list.go @@ -8,6 +8,7 @@ import ( "github.com/docker/docker/graph" "github.com/docker/docker/pkg/graphdb" + "github.com/docker/docker/utils" "github.com/docker/docker/engine" "github.com/docker/docker/pkg/parsers" @@ -131,7 +132,7 @@ func (daemon *Daemon) Containers(job *engine.Job) engine.Status { img := container.Config.Image _, tag := parsers.ParseRepositoryTag(container.Config.Image) if tag == "" { - img = img + ":" + graph.DEFAULTTAG + img = utils.ImageReference(img, graph.DEFAULTTAG) } out.SetJson("Image", img) if len(container.Args) > 0 { diff --git a/docs/man/docker-images.1.md b/docs/man/docker-images.1.md index c82d2883f7b37..c5151f1107b8c 100644 --- a/docs/man/docker-images.1.md +++ b/docs/man/docker-images.1.md @@ -8,6 +8,7 @@ docker-images - List images **docker images** [**--help**] [**-a**|**--all**[=*false*]] +[**--digests**[=*false*]] [**-f**|**--filter**[=*[]*]] [**--no-trunc**[=*false*]] [**-q**|**--quiet**[=*false*]] @@ -33,6 +34,9 @@ versions. **-a**, **--all**=*true*|*false* Show all images (by default filter out the intermediate image layers). The default is *false*. +**--digests**=*true*|*false* + Show image digests. The default is *false*. + **-f**, **--filter**=[] Filters the output. The dangling=true filter finds unused images. While label=com.foo=amd64 filters for images with a com.foo value of amd64. The label=com.foo filter finds images with the label com.foo of any value. diff --git a/docs/sources/reference/api/docker_remote_api.md b/docs/sources/reference/api/docker_remote_api.md index 10c8c62aaab8c..9ae67783ed2f1 100644 --- a/docs/sources/reference/api/docker_remote_api.md +++ b/docs/sources/reference/api/docker_remote_api.md @@ -62,6 +62,10 @@ You can set ulimit settings to be used within the container. **New!** This endpoint now returns `SystemTime`, `HttpProxy`,`HttpsProxy` and `NoProxy`. +`GET /images/json` + +**New!** +Added a `RepoDigests` field to include image digest information. ## v1.17 diff --git a/docs/sources/reference/api/docker_remote_api_v1.18.md b/docs/sources/reference/api/docker_remote_api_v1.18.md index dc5d666248156..c66fd44d32cb4 100644 --- a/docs/sources/reference/api/docker_remote_api_v1.18.md +++ b/docs/sources/reference/api/docker_remote_api_v1.18.md @@ -1054,6 +1054,45 @@ Status Codes: } ] +**Example request, with digest information**: + + GET /images/json?digests=1 HTTP/1.1 + +**Example response, with digest information**: + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Created": 1420064636, + "Id": "4986bf8c15363d1c5d15512d5266f8777bfba4974ac56e3270e7760f6f0a8125", + "ParentId": "ea13149945cb6b1e746bf28032f02e9b5a793523481a0a18645fc77ad53c4ea2", + "RepoDigests": [ + "localhost:5000/test/busybox@sha256:cbbf2f9a99b47fc460d422812b6a5adff7dfee951d8fa2e4a98caa0382cfbdbf" + ], + "RepoTags": [ + "localhost:5000/test/busybox:latest", + "playdate:latest" + ], + "Size": 0, + "VirtualSize": 2429728 + } + ] + +The response shows a single image `Id` associated with two repositories +(`RepoTags`): `localhost:5000/test/busybox`: and `playdate`. A caller can use +either of the `RepoTags` values `localhost:5000/test/busybox:latest` or +`playdate:latest` to reference the image. + +You can also use `RepoDigests` values to reference an image. In this response, +the array has only one reference and that is to the +`localhost:5000/test/busybox` repository; the `playdate` repository has no +digest. You can reference this digest using the value: +`localhost:5000/test/busybox@sha256:cbbf2f9a99b47fc460d...` + +See the `docker run` and `docker build` commands for examples of digest and tag +references on the command line. Query Parameters: diff --git a/docs/sources/reference/builder.md b/docs/sources/reference/builder.md index 9a03c516ee6b5..9ba7286a6ad43 100644 --- a/docs/sources/reference/builder.md +++ b/docs/sources/reference/builder.md @@ -192,6 +192,10 @@ Or FROM : +Or + + FROM @ + The `FROM` instruction sets the [*Base Image*](/terms/image/#base-image) for subsequent instructions. As such, a valid `Dockerfile` must have `FROM` as its first instruction. The image can be any valid image – it is especially easy @@ -204,8 +208,9 @@ to start by **pulling an image** from the [*Public Repositories*]( multiple images. Simply make a note of the last image ID output by the commit before each new `FROM` command. -If no `tag` is given to the `FROM` instruction, `latest` is assumed. If the -used tag does not exist, an error will be returned. +The `tag` or `digest` values are optional. If you omit either of them, the builder +assumes a `latest` by default. The builder returns an error if it cannot match +the `tag` value. ## MAINTAINER diff --git a/docs/sources/reference/commandline/cli.md b/docs/sources/reference/commandline/cli.md index 4ce50a7bae628..c1fcd3ea90e21 100644 --- a/docs/sources/reference/commandline/cli.md +++ b/docs/sources/reference/commandline/cli.md @@ -1112,7 +1112,9 @@ To see how the `docker:latest` image was built: List images -a, --all=false Show all images (default hides intermediate images) + --digests=false Show digests -f, --filter=[] Filter output based on conditions provided + --help=false Print usage --no-trunc=false Don't truncate output -q, --quiet=false Only show numeric IDs @@ -1161,6 +1163,22 @@ uses up the `VIRTUAL SIZE` listed only once. tryout latest 2629d1fa0b81b222fca63371ca16cbf6a0772d07759ff80e8d1369b926940074 23 hours ago 131.5 MB 5ed6274db6ceb2397844896966ea239290555e74ef307030ebb01ff91b1914df 24 hours ago 1.089 GB +#### Listing image digests + +Images that use the v2 or later format have a content-addressable identifier +called a `digest`. As long as the input used to generate the image is +unchanged, the digest value is predictable. To list image digest values, use +the `--digests` flag: + + $ sudo docker images --digests | head + REPOSITORY TAG DIGEST IMAGE ID CREATED VIRTUAL SIZE + localhost:5000/test/busybox sha256:cbbf2f9a99b47fc460d422812b6a5adff7dfee951d8fa2e4a98caa0382cfbdbf 4986bf8c1536 9 weeks ago 2.43 MB + +When pushing or pulling to a 2.0 registry, the `push` or `pull` command +output includes the image digest. You can `pull` using a digest value. You can +also reference by digest in `create`, `run`, and `rmi` commands, as well as the +`FROM` image reference in a Dockerfile. + #### Filtering The filtering flag (`-f` or `--filter`) format is of "key=value". If there is more @@ -1563,6 +1581,10 @@ use `docker pull`: $ sudo docker pull debian:testing # will pull the image named debian:testing and any intermediate # layers it is based on. + $ sudo docker pull debian@sha256:cbbf2f9a99b47fc460d422812b6a5adff7dfee951d8fa2e4a98caa0382cfbdbf + # will pull the image from the debian repository with the digest + # sha256:cbbf2f9a99b47fc460d422812b6a5adff7dfee951d8fa2e4a98caa0382cfbdbf + # and any intermediate layers it is based on. # (Typically the empty `scratch` image, a MAINTAINER layer, # and the un-tarred base). $ sudo docker pull --all-tags centos @@ -1634,9 +1656,9 @@ deleted. #### Removing tagged images -Images can be removed either by their short or long IDs, or their image -names. If an image has more than one name, each of them needs to be -removed before the image is removed. +You can remove an image using its short or long ID, its tag, or its digest. If +an image has one or more tag or digest reference, you must remove all of them +before the image is removed. $ sudo docker images REPOSITORY TAG IMAGE ID CREATED SIZE @@ -1660,6 +1682,20 @@ removed before the image is removed. Untagged: test:latest Deleted: fd484f19954f4920da7ff372b5067f5b7ddb2fd3830cecd17b96ea9e286ba5b8 +An image pulled by digest has no tag associated with it: + + $ sudo docker images --digests + REPOSITORY TAG DIGEST IMAGE ID CREATED VIRTUAL SIZE + localhost:5000/test/busybox sha256:cbbf2f9a99b47fc460d422812b6a5adff7dfee951d8fa2e4a98caa0382cfbdbf 4986bf8c1536 9 weeks ago 2.43 MB + +To remove an image using its digest: + + $ sudo docker rmi localhost:5000/test/busybox@sha256:cbbf2f9a99b47fc460d422812b6a5adff7dfee951d8fa2e4a98caa0382cfbdbf + Untagged: localhost:5000/test/busybox@sha256:cbbf2f9a99b47fc460d422812b6a5adff7dfee951d8fa2e4a98caa0382cfbdbf + Deleted: 4986bf8c15363d1c5d15512d5266f8777bfba4974ac56e3270e7760f6f0a8125 + Deleted: ea13149945cb6b1e746bf28032f02e9b5a793523481a0a18645fc77ad53c4ea2 + Deleted: df7546f9f060a2268024c8a230d8639878585defcc1bc6f79d2728a13957871b + ## run Usage: docker run [OPTIONS] IMAGE [COMMAND] [ARG...] diff --git a/docs/sources/reference/run.md b/docs/sources/reference/run.md index 44d4c04561a9b..571c0a46feb02 100644 --- a/docs/sources/reference/run.md +++ b/docs/sources/reference/run.md @@ -24,7 +24,7 @@ other `docker` command. The basic `docker run` command takes this form: - $ sudo docker run [OPTIONS] IMAGE[:TAG] [COMMAND] [ARG...] + $ sudo docker run [OPTIONS] IMAGE[:TAG|@DIGEST] [COMMAND] [ARG...] To learn how to interpret the types of `[OPTIONS]`, see [*Option types*](/reference/commandline/cli/#option-types). @@ -140,6 +140,12 @@ While not strictly a means of identifying a container, you can specify a version image you'd like to run the container with by adding `image[:tag]` to the command. For example, `docker run ubuntu:14.04`. +### Image[@digest] + +Images using the v2 or later image format have a content-addressable identifier +called a digest. As long as the input used to generate the image is unchanged, +the digest value is predictable and referenceable. + ## PID Settings (--pid) --pid="" : Set the PID (Process) Namespace mode for the container, 'host': use the host's PID namespace inside the container @@ -661,7 +667,7 @@ Dockerfile instruction and how the operator can override that setting. Recall the optional `COMMAND` in the Docker commandline: - $ sudo docker run [OPTIONS] IMAGE[:TAG] [COMMAND] [ARG...] + $ sudo docker run [OPTIONS] IMAGE[:TAG|@DIGEST] [COMMAND] [ARG...] This command is optional because the person who created the `IMAGE` may have already provided a default `COMMAND` using the Dockerfile `CMD` diff --git a/graph/history.go b/graph/history.go index 356340673f3d2..7f5063e912755 100644 --- a/graph/history.go +++ b/graph/history.go @@ -5,6 +5,7 @@ import ( "github.com/docker/docker/engine" "github.com/docker/docker/image" + "github.com/docker/docker/utils" ) func (s *TagStore) CmdHistory(job *engine.Job) engine.Status { @@ -24,7 +25,7 @@ func (s *TagStore) CmdHistory(job *engine.Job) engine.Status { if _, exists := lookupMap[id]; !exists { lookupMap[id] = []string{} } - lookupMap[id] = append(lookupMap[id], name+":"+tag) + lookupMap[id] = append(lookupMap[id], utils.ImageReference(name, tag)) } } diff --git a/graph/import.go b/graph/import.go index 3a83fcf5f41ad..44b1ecbd570ac 100644 --- a/graph/import.go +++ b/graph/import.go @@ -88,7 +88,7 @@ func (s *TagStore) CmdImport(job *engine.Job) engine.Status { job.Stdout.Write(sf.FormatStatus("", img.ID)) logID := img.ID if tag != "" { - logID += ":" + tag + logID = utils.ImageReference(logID, tag) } if err = job.Eng.Job("log", "import", logID, "").Run(); err != nil { log.Errorf("Error logging event 'import' for %s: %s", logID, err) diff --git a/graph/list.go b/graph/list.go index 9551edaae2617..9f7bccdfaac5a 100644 --- a/graph/list.go +++ b/graph/list.go @@ -1,7 +1,6 @@ package graph import ( - "fmt" "log" "path" "strings" @@ -9,6 +8,7 @@ import ( "github.com/docker/docker/engine" "github.com/docker/docker/image" "github.com/docker/docker/pkg/parsers/filters" + "github.com/docker/docker/utils" ) var acceptedImageFilterTags = map[string]struct{}{ @@ -54,22 +54,27 @@ func (s *TagStore) CmdImages(job *engine.Job) engine.Status { } lookup := make(map[string]*engine.Env) s.Lock() - for name, repository := range s.Repositories { + for repoName, repository := range s.Repositories { if job.Getenv("filter") != "" { - if match, _ := path.Match(job.Getenv("filter"), name); !match { + if match, _ := path.Match(job.Getenv("filter"), repoName); !match { continue } } - for tag, id := range repository { + for ref, id := range repository { + imgRef := utils.ImageReference(repoName, ref) image, err := s.graph.Get(id) if err != nil { - log.Printf("Warning: couldn't load %s from %s/%s: %s", id, name, tag, err) + log.Printf("Warning: couldn't load %s from %s: %s", id, imgRef, err) continue } if out, exists := lookup[id]; exists { if filt_tagged { - out.SetList("RepoTags", append(out.GetList("RepoTags"), fmt.Sprintf("%s:%s", name, tag))) + if utils.DigestReference(ref) { + out.SetList("RepoDigests", append(out.GetList("RepoDigests"), imgRef)) + } else { // Tag Ref. + out.SetList("RepoTags", append(out.GetList("RepoTags"), imgRef)) + } } } else { // get the boolean list for if only the untagged images are requested @@ -80,12 +85,20 @@ func (s *TagStore) CmdImages(job *engine.Job) engine.Status { if filt_tagged { out := &engine.Env{} out.SetJson("ParentId", image.Parent) - out.SetList("RepoTags", []string{fmt.Sprintf("%s:%s", name, tag)}) out.SetJson("Id", image.ID) out.SetInt64("Created", image.Created.Unix()) out.SetInt64("Size", image.Size) out.SetInt64("VirtualSize", image.GetParentsSize(0)+image.Size) out.SetJson("Labels", image.ContainerConfig.Labels) + + if utils.DigestReference(ref) { + out.SetList("RepoTags", []string{}) + out.SetList("RepoDigests", []string{imgRef}) + } else { + out.SetList("RepoTags", []string{imgRef}) + out.SetList("RepoDigests", []string{}) + } + lookup[id] = out } } @@ -108,6 +121,7 @@ func (s *TagStore) CmdImages(job *engine.Job) engine.Status { out := &engine.Env{} out.SetJson("ParentId", image.Parent) out.SetList("RepoTags", []string{":"}) + out.SetList("RepoDigests", []string{"@"}) out.SetJson("Id", image.ID) out.SetInt64("Created", image.Created.Unix()) out.SetInt64("Size", image.Size) diff --git a/graph/pull.go b/graph/pull.go index 8696658d1e2f3..2a6fee2f92b5d 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -22,7 +22,7 @@ import ( func (s *TagStore) CmdPull(job *engine.Job) engine.Status { if n := len(job.Args); n != 1 && n != 2 { - return job.Errorf("Usage: %s IMAGE [TAG]", job.Name) + return job.Errorf("Usage: %s IMAGE [TAG|DIGEST]", job.Name) } var ( @@ -46,7 +46,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) job.GetenvJson("metaHeaders", &metaHeaders) - c, err := s.poolAdd("pull", repoInfo.LocalName+":"+tag) + c, err := s.poolAdd("pull", utils.ImageReference(repoInfo.LocalName, tag)) if err != nil { if c != nil { // Another pull of the same repository is already taking place; just wait for it to finish @@ -56,7 +56,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { } return job.Error(err) } - defer s.poolRemove("pull", repoInfo.LocalName+":"+tag) + defer s.poolRemove("pull", utils.ImageReference(repoInfo.LocalName, tag)) log.Debugf("pulling image from host %q with remote name %q", repoInfo.Index.Name, repoInfo.RemoteName) endpoint, err := repoInfo.GetEndpoint() @@ -71,7 +71,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { logName := repoInfo.LocalName if tag != "" { - logName += ":" + tag + logName = utils.ImageReference(logName, tag) } if len(repoInfo.Index.Mirrors) == 0 && ((repoInfo.Official && repoInfo.Index.Official) || endpoint.Version == registry.APIVersion2) { @@ -113,7 +113,7 @@ func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, repoInfo * repoData, err := r.GetRepositoryData(repoInfo.RemoteName) if err != nil { if strings.Contains(err.Error(), "HTTP code: 404") { - return fmt.Errorf("Error: image %s:%s not found", repoInfo.RemoteName, askedTag) + return fmt.Errorf("Error: image %s not found", utils.ImageReference(repoInfo.RemoteName, askedTag)) } // Unexpected HTTP error return err @@ -259,7 +259,7 @@ func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, repoInfo * requestedTag := repoInfo.CanonicalName if len(askedTag) > 0 { - requestedTag = repoInfo.CanonicalName + ":" + askedTag + requestedTag = utils.ImageReference(repoInfo.CanonicalName, askedTag) } WriteStatus(requestedTag, out, sf, layers_downloaded) return nil @@ -421,7 +421,7 @@ func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out requestedTag := repoInfo.CanonicalName if len(tag) > 0 { - requestedTag = repoInfo.CanonicalName + ":" + tag + requestedTag = utils.ImageReference(repoInfo.CanonicalName, tag) } WriteStatus(requestedTag, out, sf, layersDownloaded) return nil @@ -429,7 +429,7 @@ func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, endpoint *registry.Endpoint, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool, auth *registry.RequestAuthorization) (bool, error) { log.Debugf("Pulling tag from V2 registry: %q", tag) - manifestBytes, err := r.GetV2ImageManifest(endpoint, repoInfo.RemoteName, tag, auth) + manifestBytes, digest, err := r.GetV2ImageManifest(endpoint, repoInfo.RemoteName, tag, auth) if err != nil { return false, err } @@ -444,7 +444,7 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri } if verified { - log.Printf("Image manifest for %s:%s has been verified", repoInfo.CanonicalName, tag) + log.Printf("Image manifest for %s has been verified", utils.ImageReference(repoInfo.CanonicalName, tag)) } out.Write(sf.FormatStatus(tag, "Pulling from %s", repoInfo.CanonicalName)) @@ -601,11 +601,22 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri } if verified && tagUpdated { - out.Write(sf.FormatStatus(repoInfo.CanonicalName+":"+tag, "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security.")) + out.Write(sf.FormatStatus(utils.ImageReference(repoInfo.CanonicalName, tag), "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security.")) } - if err = s.Set(repoInfo.LocalName, tag, downloads[0].img.ID, true); err != nil { - return false, err + if len(digest) > 0 { + out.Write(sf.FormatStatus("", "Digest: %s", digest)) + } + + if utils.DigestReference(tag) { + if err = s.SetDigest(repoInfo.LocalName, tag, downloads[0].img.ID); err != nil { + return false, err + } + } else { + // only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest) + if err = s.Set(repoInfo.LocalName, tag, downloads[0].img.ID, true); err != nil { + return false, err + } } return tagUpdated, nil diff --git a/graph/push.go b/graph/push.go index 117f535b50a7c..8cc2979312f38 100644 --- a/graph/push.go +++ b/graph/push.go @@ -36,8 +36,15 @@ func (s *TagStore) getImageList(localRepo map[string]string, requestedTag string for tag, id := range localRepo { if requestedTag != "" && requestedTag != tag { + // Include only the requested tag. continue } + + if utils.DigestReference(tag) { + // Ignore digest references. + continue + } + var imageListForThisTag []string tagsByImage[id] = append(tagsByImage[id], tag) @@ -76,14 +83,16 @@ func (s *TagStore) getImageList(localRepo map[string]string, requestedTag string func (s *TagStore) getImageTags(localRepo map[string]string, askedTag string) ([]string, error) { log.Debugf("Checking %s against %#v", askedTag, localRepo) if len(askedTag) > 0 { - if _, ok := localRepo[askedTag]; !ok { + if _, ok := localRepo[askedTag]; !ok || utils.DigestReference(askedTag) { return nil, fmt.Errorf("Tag does not exist: %s", askedTag) } return []string{askedTag}, nil } var tags []string for tag := range localRepo { - tags = append(tags, tag) + if !utils.DigestReference(tag) { + tags = append(tags, tag) + } } return tags, nil } @@ -422,9 +431,14 @@ func (s *TagStore) pushV2Repository(r *registry.Session, localRepo Repository, o log.Infof("Signed manifest for %s:%s using daemon's key: %s", repoInfo.LocalName, tag, s.trustKey.KeyID()) // push the manifest - if err := r.PutV2ImageManifest(endpoint, repoInfo.RemoteName, tag, bytes.NewReader(signedBody), auth); err != nil { + digest, err := r.PutV2ImageManifest(endpoint, repoInfo.RemoteName, tag, bytes.NewReader(signedBody), auth) + if err != nil { return err } + + if len(digest) > 0 { + out.Write(sf.FormatStatus("", "Digest: %s", digest)) + } } return nil } diff --git a/graph/tags.go b/graph/tags.go index 465ae7f353ead..5d26b8cfba36e 100644 --- a/graph/tags.go +++ b/graph/tags.go @@ -2,6 +2,7 @@ package graph import ( "encoding/json" + "errors" "fmt" "io/ioutil" "os" @@ -15,13 +16,16 @@ import ( "github.com/docker/docker/pkg/common" "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/registry" + "github.com/docker/docker/utils" "github.com/docker/libtrust" ) const DEFAULTTAG = "latest" var ( + //FIXME these 2 regexes also exist in registry/v2/regexp.go validTagName = regexp.MustCompile(`^[\w][\w.-]{0,127}$`) + validDigest = regexp.MustCompile(`[a-zA-Z0-9-_+.]+:[a-fA-F0-9]+`) ) type TagStore struct { @@ -107,20 +111,31 @@ func (store *TagStore) reload() error { func (store *TagStore) LookupImage(name string) (*image.Image, error) { // FIXME: standardize on returning nil when the image doesn't exist, and err for everything else // (so we can pass all errors here) - repos, tag := parsers.ParseRepositoryTag(name) - if tag == "" { - tag = DEFAULTTAG + repoName, ref := parsers.ParseRepositoryTag(name) + if ref == "" { + ref = DEFAULTTAG + } + var ( + err error + img *image.Image + ) + + img, err = store.GetImage(repoName, ref) + if err != nil { + return nil, err + } + + if img != nil { + return img, err } - img, err := store.GetImage(repos, tag) + + // name must be an image ID. store.Lock() defer store.Unlock() - if err != nil { + if img, err = store.graph.Get(name); err != nil { return nil, err - } else if img == nil { - if img, err = store.graph.Get(name); err != nil { - return nil, err - } } + return img, nil } @@ -132,7 +147,7 @@ func (store *TagStore) ByID() map[string][]string { byID := make(map[string][]string) for repoName, repository := range store.Repositories { for tag, id := range repository { - name := repoName + ":" + tag + name := utils.ImageReference(repoName, tag) if _, exists := byID[id]; !exists { byID[id] = []string{name} } else { @@ -171,32 +186,35 @@ func (store *TagStore) DeleteAll(id string) error { return nil } -func (store *TagStore) Delete(repoName, tag string) (bool, error) { +func (store *TagStore) Delete(repoName, ref string) (bool, error) { store.Lock() defer store.Unlock() deleted := false if err := store.reload(); err != nil { return false, err } + repoName = registry.NormalizeLocalName(repoName) - if r, exists := store.Repositories[repoName]; exists { - if tag != "" { - if _, exists2 := r[tag]; exists2 { - delete(r, tag) - if len(r) == 0 { - delete(store.Repositories, repoName) - } - deleted = true - } else { - return false, fmt.Errorf("No such tag: %s:%s", repoName, tag) - } - } else { + + if ref == "" { + // Delete the whole repository. + delete(store.Repositories, repoName) + return true, store.save() + } + + repoRefs, exists := store.Repositories[repoName] + if !exists { + return false, fmt.Errorf("No such repository: %s", repoName) + } + + if _, exists := repoRefs[ref]; exists { + delete(repoRefs, ref) + if len(repoRefs) == 0 { delete(store.Repositories, repoName) - deleted = true } - } else { - return false, fmt.Errorf("No such repository: %s", repoName) + deleted = true } + return deleted, store.save() } @@ -234,6 +252,40 @@ func (store *TagStore) Set(repoName, tag, imageName string, force bool) error { return store.save() } +// SetDigest creates a digest reference to an image ID. +func (store *TagStore) SetDigest(repoName, digest, imageName string) error { + img, err := store.LookupImage(imageName) + if err != nil { + return err + } + + if err := validateRepoName(repoName); err != nil { + return err + } + + if err := validateDigest(digest); err != nil { + return err + } + + store.Lock() + defer store.Unlock() + if err := store.reload(); err != nil { + return err + } + + repoName = registry.NormalizeLocalName(repoName) + repoRefs, exists := store.Repositories[repoName] + if !exists { + repoRefs = Repository{} + store.Repositories[repoName] = repoRefs + } else if oldID, exists := repoRefs[digest]; exists && oldID != img.ID { + return fmt.Errorf("Conflict: Digest %s is already set to image %s", digest, oldID) + } + + repoRefs[digest] = img.ID + return store.save() +} + func (store *TagStore) Get(repoName string) (Repository, error) { store.Lock() defer store.Unlock() @@ -247,24 +299,29 @@ func (store *TagStore) Get(repoName string) (Repository, error) { return nil, nil } -func (store *TagStore) GetImage(repoName, tagOrID string) (*image.Image, error) { +func (store *TagStore) GetImage(repoName, refOrID string) (*image.Image, error) { repo, err := store.Get(repoName) - store.Lock() - defer store.Unlock() + if err != nil { return nil, err - } else if repo == nil { + } + if repo == nil { return nil, nil } - if revision, exists := repo[tagOrID]; exists { - return store.graph.Get(revision) + + store.Lock() + defer store.Unlock() + if imgID, exists := repo[refOrID]; exists { + return store.graph.Get(imgID) } + // If no matching tag is found, search through images for a matching image id for _, revision := range repo { - if strings.HasPrefix(revision, tagOrID) { + if strings.HasPrefix(revision, refOrID) { return store.graph.Get(revision) } } + return nil, nil } @@ -275,7 +332,7 @@ func (store *TagStore) GetRepoRefs() map[string][]string { for name, repository := range store.Repositories { for tag, id := range repository { shortID := common.TruncateID(id) - reporefs[shortID] = append(reporefs[shortID], fmt.Sprintf("%s:%s", name, tag)) + reporefs[shortID] = append(reporefs[shortID], utils.ImageReference(name, tag)) } } store.Unlock() @@ -293,10 +350,10 @@ func validateRepoName(name string) error { return nil } -// Validate the name of a tag +// ValidateTagName validates the name of a tag func ValidateTagName(name string) error { if name == "" { - return fmt.Errorf("Tag name can't be empty") + return fmt.Errorf("tag name can't be empty") } if !validTagName.MatchString(name) { return fmt.Errorf("Illegal tag name (%s): only [A-Za-z0-9_.-] are allowed, minimum 1, maximum 128 in length", name) @@ -304,6 +361,16 @@ func ValidateTagName(name string) error { return nil } +func validateDigest(dgst string) error { + if dgst == "" { + return errors.New("digest can't be empty") + } + if !validDigest.MatchString(dgst) { + return fmt.Errorf("illegal digest (%s): must be of the form [a-zA-Z0-9-_+.]+:[a-fA-F0-9]+", dgst) + } + return nil +} + func (store *TagStore) poolAdd(kind, key string) (chan struct{}, error) { store.Lock() defer store.Unlock() diff --git a/graph/tags_unit_test.go b/graph/tags_unit_test.go index 58ad8ed878345..c1a686bbc416f 100644 --- a/graph/tags_unit_test.go +++ b/graph/tags_unit_test.go @@ -21,6 +21,8 @@ const ( testPrivateImageName = "127.0.0.1:8000/privateapp" testPrivateImageID = "5bc255f8699e4ee89ac4469266c3d11515da88fdcbde45d7b069b636ff4efd81" testPrivateImageIDShort = "5bc255f8699e" + testPrivateImageDigest = "sha256:bc8813ea7b3603864987522f02a76101c17ad122e1c46d790efc0fca78ca7bfb" + testPrivateImageTag = "sometag" ) func fakeTar() (io.Reader, error) { @@ -83,6 +85,9 @@ func mkTestTagStore(root string, t *testing.T) *TagStore { if err := store.Set(testPrivateImageName, "", testPrivateImageID, false); err != nil { t.Fatal(err) } + if err := store.SetDigest(testPrivateImageName, testPrivateImageDigest, testPrivateImageID); err != nil { + t.Fatal(err) + } return store } @@ -128,6 +133,10 @@ func TestLookupImage(t *testing.T) { "fail:fail", } + digestLookups := []string{ + testPrivateImageName + "@" + testPrivateImageDigest, + } + for _, name := range officialLookups { if img, err := store.LookupImage(name); err != nil { t.Errorf("Error looking up %s: %s", name, err) @@ -155,6 +164,16 @@ func TestLookupImage(t *testing.T) { t.Errorf("Expected 0 image, 1 found: %s", name) } } + + for _, name := range digestLookups { + if img, err := store.LookupImage(name); err != nil { + t.Errorf("Error looking up %s: %s", name, err) + } else if img == nil { + t.Errorf("Expected 1 image, none found: %s", name) + } else if img.ID != testPrivateImageID { + t.Errorf("Expected ID '%s' found '%s'", testPrivateImageID, img.ID) + } + } } func TestValidTagName(t *testing.T) { @@ -174,3 +193,24 @@ func TestInvalidTagName(t *testing.T) { } } } + +func TestValidateDigest(t *testing.T) { + tests := []struct { + input string + expectError bool + }{ + {"", true}, + {"latest", true}, + {"a:b", false}, + {"aZ0124-.+:bY852-_.+=", false}, + {"#$%#$^:$%^#$%", true}, + } + + for i, test := range tests { + err := validateDigest(test.input) + gotError := err != nil + if e, a := test.expectError, gotError; e != a { + t.Errorf("%d: with input %s, expected error=%t, got %t: %s", i, test.input, test.expectError, gotError, err) + } + } +} diff --git a/integration-cli/docker_cli_by_digest_test.go b/integration-cli/docker_cli_by_digest_test.go new file mode 100644 index 0000000000000..24ebf0cf70b7c --- /dev/null +++ b/integration-cli/docker_cli_by_digest_test.go @@ -0,0 +1,535 @@ +package main + +import ( + "fmt" + "os/exec" + "regexp" + "strings" + "testing" + + "github.com/docker/docker/utils" +) + +var ( + repoName = fmt.Sprintf("%v/dockercli/busybox-by-dgst", privateRegistryURL) + digestRegex = regexp.MustCompile("Digest: ([^\n]+)") +) + +func setupImage() (string, error) { + return setupImageWithTag("latest") +} + +func setupImageWithTag(tag string) (string, error) { + containerName := "busyboxbydigest" + + c := exec.Command(dockerBinary, "run", "-d", "-e", "digest=1", "--name", containerName, "busybox") + if _, err := runCommand(c); err != nil { + return "", err + } + + // tag the image to upload it to the private registry + repoAndTag := utils.ImageReference(repoName, tag) + c = exec.Command(dockerBinary, "commit", containerName, repoAndTag) + if out, _, err := runCommandWithOutput(c); err != nil { + return "", fmt.Errorf("image tagging failed: %s, %v", out, err) + } + defer deleteImages(repoAndTag) + + // delete the container as we don't need it any more + if err := deleteContainer(containerName); err != nil { + return "", err + } + + // push the image + c = exec.Command(dockerBinary, "push", repoAndTag) + out, _, err := runCommandWithOutput(c) + if err != nil { + return "", fmt.Errorf("pushing the image to the private registry has failed: %s, %v", out, err) + } + + // delete our local repo that we previously tagged + c = exec.Command(dockerBinary, "rmi", repoAndTag) + if out, _, err := runCommandWithOutput(c); err != nil { + return "", fmt.Errorf("error deleting images prior to real test: %s, %v", out, err) + } + + // the push output includes "Digest: ", so find that + matches := digestRegex.FindStringSubmatch(out) + if len(matches) != 2 { + return "", fmt.Errorf("unable to parse digest from push output: %s", out) + } + pushDigest := matches[1] + + return pushDigest, nil +} + +func TestPullByTagDisplaysDigest(t *testing.T) { + defer setupRegistry(t)() + + pushDigest, err := setupImage() + if err != nil { + t.Fatalf("error setting up image: %v", err) + } + + // pull from the registry using the tag + c := exec.Command(dockerBinary, "pull", repoName) + out, _, err := runCommandWithOutput(c) + if err != nil { + t.Fatalf("error pulling by tag: %s, %v", out, err) + } + defer deleteImages(repoName) + + // the pull output includes "Digest: ", so find that + matches := digestRegex.FindStringSubmatch(out) + if len(matches) != 2 { + t.Fatalf("unable to parse digest from pull output: %s", out) + } + pullDigest := matches[1] + + // make sure the pushed and pull digests match + if pushDigest != pullDigest { + t.Fatalf("push digest %q didn't match pull digest %q", pushDigest, pullDigest) + } + + logDone("by_digest - pull by tag displays digest") +} + +func TestPullByDigest(t *testing.T) { + defer setupRegistry(t)() + + pushDigest, err := setupImage() + if err != nil { + t.Fatalf("error setting up image: %v", err) + } + + // pull from the registry using the @ reference + imageReference := fmt.Sprintf("%s@%s", repoName, pushDigest) + c := exec.Command(dockerBinary, "pull", imageReference) + out, _, err := runCommandWithOutput(c) + if err != nil { + t.Fatalf("error pulling by digest: %s, %v", out, err) + } + defer deleteImages(imageReference) + + // the pull output includes "Digest: ", so find that + matches := digestRegex.FindStringSubmatch(out) + if len(matches) != 2 { + t.Fatalf("unable to parse digest from pull output: %s", out) + } + pullDigest := matches[1] + + // make sure the pushed and pull digests match + if pushDigest != pullDigest { + t.Fatalf("push digest %q didn't match pull digest %q", pushDigest, pullDigest) + } + + logDone("by_digest - pull by digest") +} + +func TestCreateByDigest(t *testing.T) { + defer setupRegistry(t)() + + pushDigest, err := setupImage() + if err != nil { + t.Fatalf("error setting up image: %v", err) + } + + imageReference := fmt.Sprintf("%s@%s", repoName, pushDigest) + + containerName := "createByDigest" + c := exec.Command(dockerBinary, "create", "--name", containerName, imageReference) + out, _, err := runCommandWithOutput(c) + if err != nil { + t.Fatalf("error creating by digest: %s, %v", out, err) + } + defer deleteContainer(containerName) + + res, err := inspectField(containerName, "Config.Image") + if err != nil { + t.Fatalf("failed to get Config.Image: %s, %v", out, err) + } + if res != imageReference { + t.Fatalf("unexpected Config.Image: %s (expected %s)", res, imageReference) + } + + logDone("by_digest - create by digest") +} + +func TestRunByDigest(t *testing.T) { + defer setupRegistry(t)() + + pushDigest, err := setupImage() + if err != nil { + t.Fatalf("error setting up image: %v", err) + } + + imageReference := fmt.Sprintf("%s@%s", repoName, pushDigest) + + containerName := "runByDigest" + c := exec.Command(dockerBinary, "run", "--name", containerName, imageReference, "sh", "-c", "echo found=$digest") + out, _, err := runCommandWithOutput(c) + if err != nil { + t.Fatalf("error run by digest: %s, %v", out, err) + } + defer deleteContainer(containerName) + + foundRegex := regexp.MustCompile("found=([^\n]+)") + matches := foundRegex.FindStringSubmatch(out) + if len(matches) != 2 { + t.Fatalf("error locating expected 'found=1' output: %s", out) + } + if matches[1] != "1" { + t.Fatalf("Expected %q, got %q", "1", matches[1]) + } + + res, err := inspectField(containerName, "Config.Image") + if err != nil { + t.Fatalf("failed to get Config.Image: %s, %v", out, err) + } + if res != imageReference { + t.Fatalf("unexpected Config.Image: %s (expected %s)", res, imageReference) + } + + logDone("by_digest - run by digest") +} + +func TestRemoveImageByDigest(t *testing.T) { + defer setupRegistry(t)() + + digest, err := setupImage() + if err != nil { + t.Fatalf("error setting up image: %v", err) + } + + imageReference := fmt.Sprintf("%s@%s", repoName, digest) + + // pull from the registry using the @ reference + c := exec.Command(dockerBinary, "pull", imageReference) + out, _, err := runCommandWithOutput(c) + if err != nil { + t.Fatalf("error pulling by digest: %s, %v", out, err) + } + + // make sure inspect runs ok + if _, err := inspectField(imageReference, "Id"); err != nil { + t.Fatalf("failed to inspect image: %v", err) + } + + // do the delete + if err := deleteImages(imageReference); err != nil { + t.Fatalf("unexpected error deleting image: %v", err) + } + + // try to inspect again - it should error this time + if _, err := inspectField(imageReference, "Id"); err == nil { + t.Fatalf("unexpected nil err trying to inspect what should be a non-existent image") + } else if !strings.Contains(err.Error(), "No such image") { + t.Fatalf("expected 'No such image' output, got %v", err) + } + + logDone("by_digest - remove image by digest") +} + +func TestBuildByDigest(t *testing.T) { + defer setupRegistry(t)() + + digest, err := setupImage() + if err != nil { + t.Fatalf("error setting up image: %v", err) + } + + imageReference := fmt.Sprintf("%s@%s", repoName, digest) + + // pull from the registry using the @ reference + c := exec.Command(dockerBinary, "pull", imageReference) + out, _, err := runCommandWithOutput(c) + if err != nil { + t.Fatalf("error pulling by digest: %s, %v", out, err) + } + + // get the image id + imageID, err := inspectField(imageReference, "Id") + if err != nil { + t.Fatalf("error getting image id: %v", err) + } + + // do the build + name := "buildbydigest" + defer deleteImages(name) + _, err = buildImage(name, fmt.Sprintf( + `FROM %s + CMD ["/bin/echo", "Hello World"]`, imageReference), + true) + if err != nil { + t.Fatal(err) + } + + // get the build's image id + res, err := inspectField(name, "Config.Image") + if err != nil { + t.Fatal(err) + } + // make sure they match + if res != imageID { + t.Fatalf("Image %s, expected %s", res, imageID) + } + + logDone("by_digest - build by digest") +} + +func TestTagByDigest(t *testing.T) { + defer setupRegistry(t)() + + digest, err := setupImage() + if err != nil { + t.Fatalf("error setting up image: %v", err) + } + + imageReference := fmt.Sprintf("%s@%s", repoName, digest) + + // pull from the registry using the @ reference + c := exec.Command(dockerBinary, "pull", imageReference) + out, _, err := runCommandWithOutput(c) + if err != nil { + t.Fatalf("error pulling by digest: %s, %v", out, err) + } + + // tag it + tag := "tagbydigest" + c = exec.Command(dockerBinary, "tag", imageReference, tag) + if _, err := runCommand(c); err != nil { + t.Fatalf("unexpected error tagging: %v", err) + } + + expectedID, err := inspectField(imageReference, "Id") + if err != nil { + t.Fatalf("error getting original image id: %v", err) + } + + tagID, err := inspectField(tag, "Id") + if err != nil { + t.Fatalf("error getting tagged image id: %v", err) + } + + if tagID != expectedID { + t.Fatalf("expected image id %q, got %q", expectedID, tagID) + } + + logDone("by_digest - tag by digest") +} + +func TestListImagesWithoutDigests(t *testing.T) { + defer setupRegistry(t)() + + digest, err := setupImage() + if err != nil { + t.Fatalf("error setting up image: %v", err) + } + + imageReference := fmt.Sprintf("%s@%s", repoName, digest) + + // pull from the registry using the @ reference + c := exec.Command(dockerBinary, "pull", imageReference) + out, _, err := runCommandWithOutput(c) + if err != nil { + t.Fatalf("error pulling by digest: %s, %v", out, err) + } + + c = exec.Command(dockerBinary, "images") + out, _, err = runCommandWithOutput(c) + if err != nil { + t.Fatalf("error listing images: %s, %v", out, err) + } + + if strings.Contains(out, "DIGEST") { + t.Fatalf("list output should not have contained DIGEST header: %s", out) + } + + logDone("by_digest - list images - digest header not displayed by default") +} + +func TestListImagesWithDigests(t *testing.T) { + defer setupRegistry(t)() + defer deleteImages(repoName+":tag1", repoName+":tag2") + + // setup image1 + digest1, err := setupImageWithTag("tag1") + if err != nil { + t.Fatalf("error setting up image: %v", err) + } + imageReference1 := fmt.Sprintf("%s@%s", repoName, digest1) + defer deleteImages(imageReference1) + t.Logf("imageReference1 = %s", imageReference1) + + // pull image1 by digest + c := exec.Command(dockerBinary, "pull", imageReference1) + out, _, err := runCommandWithOutput(c) + if err != nil { + t.Fatalf("error pulling by digest: %s, %v", out, err) + } + + // list images + c = exec.Command(dockerBinary, "images", "--digests") + out, _, err = runCommandWithOutput(c) + if err != nil { + t.Fatalf("error listing images: %s, %v", out, err) + } + + // make sure repo shown, tag=, digest = $digest1 + re1 := regexp.MustCompile(`\s*` + repoName + `\s*\s*` + digest1 + `\s`) + if !re1.MatchString(out) { + t.Fatalf("expected %q: %s", re1.String(), out) + } + + // setup image2 + digest2, err := setupImageWithTag("tag2") + if err != nil { + t.Fatalf("error setting up image: %v", err) + } + imageReference2 := fmt.Sprintf("%s@%s", repoName, digest2) + defer deleteImages(imageReference2) + t.Logf("imageReference2 = %s", imageReference2) + + // pull image1 by digest + c = exec.Command(dockerBinary, "pull", imageReference1) + out, _, err = runCommandWithOutput(c) + if err != nil { + t.Fatalf("error pulling by digest: %s, %v", out, err) + } + + // pull image2 by digest + c = exec.Command(dockerBinary, "pull", imageReference2) + out, _, err = runCommandWithOutput(c) + if err != nil { + t.Fatalf("error pulling by digest: %s, %v", out, err) + } + + // list images + c = exec.Command(dockerBinary, "images", "--digests") + out, _, err = runCommandWithOutput(c) + if err != nil { + t.Fatalf("error listing images: %s, %v", out, err) + } + + // make sure repo shown, tag=, digest = $digest1 + if !re1.MatchString(out) { + t.Fatalf("expected %q: %s", re1.String(), out) + } + + // make sure repo shown, tag=, digest = $digest2 + re2 := regexp.MustCompile(`\s*` + repoName + `\s*\s*` + digest2 + `\s`) + if !re2.MatchString(out) { + t.Fatalf("expected %q: %s", re2.String(), out) + } + + // pull tag1 + c = exec.Command(dockerBinary, "pull", repoName+":tag1") + out, _, err = runCommandWithOutput(c) + if err != nil { + t.Fatalf("error pulling tag1: %s, %v", out, err) + } + + // list images + c = exec.Command(dockerBinary, "images", "--digests") + out, _, err = runCommandWithOutput(c) + if err != nil { + t.Fatalf("error listing images: %s, %v", out, err) + } + + // make sure image 1 has repo, tag, AND repo, , digest + reWithTag1 := regexp.MustCompile(`\s*` + repoName + `\s*tag1\s*\s`) + reWithDigest1 := regexp.MustCompile(`\s*` + repoName + `\s*\s*` + digest1 + `\s`) + if !reWithTag1.MatchString(out) { + t.Fatalf("expected %q: %s", reWithTag1.String(), out) + } + if !reWithDigest1.MatchString(out) { + t.Fatalf("expected %q: %s", reWithDigest1.String(), out) + } + // make sure image 2 has repo, , digest + if !re2.MatchString(out) { + t.Fatalf("expected %q: %s", re2.String(), out) + } + + // pull tag 2 + c = exec.Command(dockerBinary, "pull", repoName+":tag2") + out, _, err = runCommandWithOutput(c) + if err != nil { + t.Fatalf("error pulling tag2: %s, %v", out, err) + } + + // list images + c = exec.Command(dockerBinary, "images", "--digests") + out, _, err = runCommandWithOutput(c) + if err != nil { + t.Fatalf("error listing images: %s, %v", out, err) + } + + // make sure image 1 has repo, tag, digest + if !reWithTag1.MatchString(out) { + t.Fatalf("expected %q: %s", re1.String(), out) + } + + // make sure image 2 has repo, tag, digest + reWithTag2 := regexp.MustCompile(`\s*` + repoName + `\s*tag2\s*\s`) + reWithDigest2 := regexp.MustCompile(`\s*` + repoName + `\s*\s*` + digest2 + `\s`) + if !reWithTag2.MatchString(out) { + t.Fatalf("expected %q: %s", reWithTag2.String(), out) + } + if !reWithDigest2.MatchString(out) { + t.Fatalf("expected %q: %s", reWithDigest2.String(), out) + } + + // list images + c = exec.Command(dockerBinary, "images", "--digests") + out, _, err = runCommandWithOutput(c) + if err != nil { + t.Fatalf("error listing images: %s, %v", out, err) + } + + // make sure image 1 has repo, tag, digest + if !reWithTag1.MatchString(out) { + t.Fatalf("expected %q: %s", re1.String(), out) + } + // make sure image 2 has repo, tag, digest + if !reWithTag2.MatchString(out) { + t.Fatalf("expected %q: %s", re2.String(), out) + } + // make sure busybox has tag, but not digest + busyboxRe := regexp.MustCompile(`\s*busybox\s*latest\s*\s`) + if !busyboxRe.MatchString(out) { + t.Fatalf("expected %q: %s", busyboxRe.String(), out) + } + + logDone("by_digest - list images with digests") +} + +func TestDeleteImageByIDOnlyPulledByDigest(t *testing.T) { + defer setupRegistry(t)() + + pushDigest, err := setupImage() + if err != nil { + t.Fatalf("error setting up image: %v", err) + } + + // pull from the registry using the @ reference + imageReference := fmt.Sprintf("%s@%s", repoName, pushDigest) + c := exec.Command(dockerBinary, "pull", imageReference) + out, _, err := runCommandWithOutput(c) + if err != nil { + t.Fatalf("error pulling by digest: %s, %v", out, err) + } + // just in case... + defer deleteImages(imageReference) + + imageID, err := inspectField(imageReference, ".Id") + if err != nil { + t.Fatalf("error inspecting image id: %v", err) + } + + c = exec.Command(dockerBinary, "rmi", imageID) + if _, err := runCommand(c); err != nil { + t.Fatalf("error deleting image by id: %v", err) + } + + logDone("by_digest - delete image by id only pulled by digest") +} diff --git a/integration-cli/docker_cli_push_test.go b/integration-cli/docker_cli_push_test.go index 526eb19ac553b..f1274ba706e55 100644 --- a/integration-cli/docker_cli_push_test.go +++ b/integration-cli/docker_cli_push_test.go @@ -17,7 +17,7 @@ func TestPushBusyboxImage(t *testing.T) { defer setupRegistry(t)() repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) - // tag the image to upload it tot he private registry + // tag the image to upload it to the private registry tagCmd := exec.Command(dockerBinary, "tag", "busybox", repoName) if out, _, err := runCommandWithOutput(tagCmd); err != nil { t.Fatalf("image tagging failed: %s, %v", out, err) diff --git a/pkg/parsers/parsers.go b/pkg/parsers/parsers.go index 6563190410507..59e294dc2255b 100644 --- a/pkg/parsers/parsers.go +++ b/pkg/parsers/parsers.go @@ -62,11 +62,17 @@ func ParseTCPAddr(addr string, defaultAddr string) (string, error) { return fmt.Sprintf("tcp://%s:%d", host, p), nil } -// Get a repos name and returns the right reposName + tag +// Get a repos name and returns the right reposName + tag|digest // The tag can be confusing because of a port in a repository name. // Ex: localhost.localdomain:5000/samalba/hipache:latest +// Digest ex: localhost:5000/foo/bar@sha256:bc8813ea7b3603864987522f02a76101c17ad122e1c46d790efc0fca78ca7bfb func ParseRepositoryTag(repos string) (string, string) { - n := strings.LastIndex(repos, ":") + n := strings.Index(repos, "@") + if n >= 0 { + parts := strings.Split(repos, "@") + return parts[0], parts[1] + } + n = strings.LastIndex(repos, ":") if n < 0 { return repos, "" } diff --git a/pkg/parsers/parsers_test.go b/pkg/parsers/parsers_test.go index aac1e33e35a57..bc9a1e943c2ed 100644 --- a/pkg/parsers/parsers_test.go +++ b/pkg/parsers/parsers_test.go @@ -49,18 +49,27 @@ func TestParseRepositoryTag(t *testing.T) { if repo, tag := ParseRepositoryTag("root:tag"); repo != "root" || tag != "tag" { t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "root", "tag", repo, tag) } + if repo, digest := ParseRepositoryTag("root@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); repo != "root" || digest != "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" { + t.Errorf("Expected repo: '%s' and digest: '%s', got '%s' and '%s'", "root", "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", repo, digest) + } if repo, tag := ParseRepositoryTag("user/repo"); repo != "user/repo" || tag != "" { t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "user/repo", "", repo, tag) } if repo, tag := ParseRepositoryTag("user/repo:tag"); repo != "user/repo" || tag != "tag" { t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "user/repo", "tag", repo, tag) } + if repo, digest := ParseRepositoryTag("user/repo@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); repo != "user/repo" || digest != "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" { + t.Errorf("Expected repo: '%s' and digest: '%s', got '%s' and '%s'", "user/repo", "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", repo, digest) + } if repo, tag := ParseRepositoryTag("url:5000/repo"); repo != "url:5000/repo" || tag != "" { t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "url:5000/repo", "", repo, tag) } if repo, tag := ParseRepositoryTag("url:5000/repo:tag"); repo != "url:5000/repo" || tag != "tag" { t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "url:5000/repo", "tag", repo, tag) } + if repo, digest := ParseRepositoryTag("url:5000/repo@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); repo != "url:5000/repo" || digest != "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" { + t.Errorf("Expected repo: '%s' and digest: '%s', got '%s' and '%s'", "url:5000/repo", "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", repo, digest) + } } func TestParsePortMapping(t *testing.T) { diff --git a/registry/session_v2.go b/registry/session_v2.go index da5371d83bfbb..c5bee11bc6647 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -12,6 +12,8 @@ import ( "github.com/docker/docker/utils" ) +const DockerDigestHeader = "Docker-Content-Digest" + func getV2Builder(e *Endpoint) *v2.URLBuilder { if e.URLBuilder == nil { e.URLBuilder = v2.NewURLBuilder(e.URL) @@ -63,10 +65,10 @@ func (r *Session) GetV2Authorization(ep *Endpoint, imageName string, readOnly bo // 1.c) if anything else, err // 2) PUT the created/signed manifest // -func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, auth *RequestAuthorization) ([]byte, error) { +func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, auth *RequestAuthorization) ([]byte, string, error) { routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) if err != nil { - return nil, err + return nil, "", err } method := "GET" @@ -74,30 +76,30 @@ func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, au req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { - return nil, err + return nil, "", err } if err := auth.Authorize(req); err != nil { - return nil, err + return nil, "", err } res, _, err := r.doRequest(req) if err != nil { - return nil, err + return nil, "", err } defer res.Body.Close() if res.StatusCode != 200 { if res.StatusCode == 401 { - return nil, errLoginRequired + return nil, "", errLoginRequired } else if res.StatusCode == 404 { - return nil, ErrDoesNotExist + return nil, "", ErrDoesNotExist } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res) + return nil, "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res) } buf, err := ioutil.ReadAll(res.Body) if err != nil { - return nil, fmt.Errorf("Error while reading the http response: %s", err) + return nil, "", fmt.Errorf("Error while reading the http response: %s", err) } - return buf, nil + return buf, res.Header.Get(DockerDigestHeader), nil } // - Succeeded to head image blob (already exists) @@ -261,41 +263,41 @@ func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string } // Finally Push the (signed) manifest of the blobs we've just pushed -func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) error { +func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) (string, error) { routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) if err != nil { - return err + return "", err } method := "PUT" log.Debugf("[registry] Calling %q %s", method, routeURL) req, err := r.reqFactory.NewRequest(method, routeURL, manifestRdr) if err != nil { - return err + return "", err } if err := auth.Authorize(req); err != nil { - return err + return "", err } res, _, err := r.doRequest(req) if err != nil { - return err + return "", err } defer res.Body.Close() // All 2xx and 3xx responses can be accepted for a put. if res.StatusCode >= 400 { if res.StatusCode == 401 { - return errLoginRequired + return "", errLoginRequired } errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return err + return "", err } log.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) - return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) + return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) } - return nil + return res.Header.Get(DockerDigestHeader), nil } type remoteTags struct { diff --git a/registry/v2/regexp.go b/registry/v2/regexp.go index e1e923b99e46e..07484dcd69538 100644 --- a/registry/v2/regexp.go +++ b/registry/v2/regexp.go @@ -17,3 +17,6 @@ var RepositoryNameRegexp = regexp.MustCompile(`(?:` + RepositoryNameComponentReg // TagNameRegexp matches valid tag names. From docker/docker:graph/tags.go. var TagNameRegexp = regexp.MustCompile(`[\w][\w.-]{0,127}`) + +// DigestRegexp matches valid digest types. +var DigestRegexp = regexp.MustCompile(`[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+`) diff --git a/registry/v2/routes.go b/registry/v2/routes.go index 08f36e2f712bb..de0a38fb815a5 100644 --- a/registry/v2/routes.go +++ b/registry/v2/routes.go @@ -33,11 +33,11 @@ func Router() *mux.Router { Path("/v2/"). Name(RouteNameBase) - // GET /v2//manifest/ Image Manifest Fetch the image manifest identified by name and tag. - // PUT /v2//manifest/ Image Manifest Upload the image manifest identified by name and tag. - // DELETE /v2//manifest/ Image Manifest Delete the image identified by name and tag. + // GET /v2//manifest/ Image Manifest Fetch the image manifest identified by name and reference where reference can be a tag or digest. + // PUT /v2//manifest/ Image Manifest Upload the image manifest identified by name and reference where reference can be a tag or digest. + // DELETE /v2//manifest/ Image Manifest Delete the image identified by name and reference where reference can be a tag or digest. router. - Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/manifests/{tag:" + TagNameRegexp.String() + "}"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/manifests/{reference:" + TagNameRegexp.String() + "|" + DigestRegexp.String() + "}"). Name(RouteNameManifest) // GET /v2//tags/list Tags Fetch the tags under the repository identified by name. diff --git a/registry/v2/routes_test.go b/registry/v2/routes_test.go index 7682792e04fb8..0191feed00189 100644 --- a/registry/v2/routes_test.go +++ b/registry/v2/routes_test.go @@ -55,16 +55,16 @@ func TestRouter(t *testing.T) { RouteName: RouteNameManifest, RequestURI: "/v2/foo/manifests/bar", Vars: map[string]string{ - "name": "foo", - "tag": "bar", + "name": "foo", + "reference": "bar", }, }, { RouteName: RouteNameManifest, RequestURI: "/v2/foo/bar/manifests/tag", Vars: map[string]string{ - "name": "foo/bar", - "tag": "tag", + "name": "foo/bar", + "reference": "tag", }, }, { @@ -128,8 +128,8 @@ func TestRouter(t *testing.T) { RouteName: RouteNameManifest, RequestURI: "/v2/foo/bar/manifests/manifests/tags", Vars: map[string]string{ - "name": "foo/bar/manifests", - "tag": "tags", + "name": "foo/bar/manifests", + "reference": "tags", }, }, { diff --git a/registry/v2/urls.go b/registry/v2/urls.go index d1380b47abcf1..38fa98af01d33 100644 --- a/registry/v2/urls.go +++ b/registry/v2/urls.go @@ -74,11 +74,11 @@ func (ub *URLBuilder) BuildTagsURL(name string) (string, error) { return tagsURL.String(), nil } -// BuildManifestURL constructs a url for the manifest identified by name and tag. -func (ub *URLBuilder) BuildManifestURL(name, tag string) (string, error) { +// BuildManifestURL constructs a url for the manifest identified by name and reference. +func (ub *URLBuilder) BuildManifestURL(name, reference string) (string, error) { route := ub.cloneRoute(RouteNameManifest) - manifestURL, err := route.URL("name", name, "tag", tag) + manifestURL, err := route.URL("name", name, "reference", reference) if err != nil { return "", err } diff --git a/utils/utils.go b/utils/utils.go index cc3b499f65d99..540ae6f57275b 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -535,3 +535,20 @@ func (wc *WriteCounter) Write(p []byte) (count int, err error) { wc.Count += int64(count) return } + +// ImageReference combines `repo` and `ref` and returns a string representing +// the combination. If `ref` is a digest (meaning it's of the form +// :, the returned string is @. Otherwise, +// ref is assumed to be a tag, and the returned string is :. +func ImageReference(repo, ref string) string { + if DigestReference(ref) { + return repo + "@" + ref + } + return repo + ":" + ref +} + +// DigestReference returns true if ref is a digest reference; i.e. if it +// is of the form :. +func DigestReference(ref string) bool { + return strings.Contains(ref, ":") +} diff --git a/utils/utils_test.go b/utils/utils_test.go index ef1f7af03b410..94303a0e96819 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -122,3 +122,33 @@ func TestWriteCounter(t *testing.T) { t.Error("Wrong message written") } } + +func TestImageReference(t *testing.T) { + tests := []struct { + repo string + ref string + expected string + }{ + {"repo", "tag", "repo:tag"}, + {"repo", "sha256:c100b11b25d0cacd52c14e0e7bf525e1a4c0e6aec8827ae007055545909d1a64", "repo@sha256:c100b11b25d0cacd52c14e0e7bf525e1a4c0e6aec8827ae007055545909d1a64"}, + } + + for i, test := range tests { + actual := ImageReference(test.repo, test.ref) + if test.expected != actual { + t.Errorf("%d: expected %q, got %q", i, test.expected, actual) + } + } +} + +func TestDigestReference(t *testing.T) { + input := "sha256:c100b11b25d0cacd52c14e0e7bf525e1a4c0e6aec8827ae007055545909d1a64" + if !DigestReference(input) { + t.Errorf("Expected DigestReference=true for input %q", input) + } + + input = "latest" + if DigestReference(input) { + t.Errorf("Unexpected DigestReference=true for input %q", input) + } +}