From 359d0c247fe523ef82676f9ae4046b79e6854fe1 Mon Sep 17 00:00:00 2001 From: Jessica Frazelle Date: Mon, 31 Aug 2015 10:06:22 -0700 Subject: [PATCH] update download-frozen-image.sh to v2 registry Signed-off-by: Jessica Frazelle --- Dockerfile | 11 +- ...n-image.sh => download-frozen-image-v1.sh} | 0 contrib/download-frozen-image-v2.sh | 113 ++++++++++++++++++ hack/make/.ensure-frozen-images | 40 ++++--- integration-cli/docker_cli_run_test.go | 2 +- 5 files changed, 141 insertions(+), 25 deletions(-) rename contrib/{download-frozen-image.sh => download-frozen-image-v1.sh} (100%) create mode 100755 contrib/download-frozen-image-v2.sh diff --git a/Dockerfile b/Dockerfile index 3ff6b9080e038..29f4f440d2303 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,6 +49,7 @@ RUN apt-get update && apt-get install -y \ gcc-mingw-w64 \ git \ iptables \ + jq \ libapparmor-dev \ libcap-dev \ libltdl-dev \ @@ -175,11 +176,11 @@ RUN ln -sfv $PWD/.bashrc ~/.bashrc RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker # Get useful and necessary Hub images so we can "docker load" locally instead of pulling -COPY contrib/download-frozen-image.sh /go/src/github.com/docker/docker/contrib/ -RUN ./contrib/download-frozen-image.sh /docker-frozen-images \ - busybox:latest@d7057cb020844f245031d27b76cb18af05db1cc3a96a29fa7777af75f5ac91a3 \ - hello-world:frozen@91c95931e552b11604fea91c2f537284149ec32fff0f700a4769cfd31d7696ae \ - jess/unshare@5c9f6ea50341a2a8eb6677527f2bdedbf331ae894a41714fda770fb130f3314d +COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ +RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ + busybox:latest@sha256:eb3c0d4680f9213ee5f348ea6d39489a1f85a318a2ae09e012c426f78252a6d2 \ + hello-world:latest@sha256:8be990ef2aeb16dbcb9271ddfe2610fa6658d13f6dfb8bc72074cc1ca36966a7 \ + jess/unshare:latest@sha256:2e3a8c0591c4690b82d4eba7e5ef8f49f2ddfe9f867f3e865198db9bd1436c5b # see also "hack/make/.ensure-frozen-images" (which needs to be updated any time this list is) # Download man page generator diff --git a/contrib/download-frozen-image.sh b/contrib/download-frozen-image-v1.sh similarity index 100% rename from contrib/download-frozen-image.sh rename to contrib/download-frozen-image-v1.sh diff --git a/contrib/download-frozen-image-v2.sh b/contrib/download-frozen-image-v2.sh new file mode 100755 index 0000000000000..196d4b532a0ba --- /dev/null +++ b/contrib/download-frozen-image-v2.sh @@ -0,0 +1,113 @@ +#!/bin/bash +set -e + +# hello-world latest ef872312fe1b 3 months ago 910 B +# hello-world latest ef872312fe1bbc5e05aae626791a47ee9b032efa8f3bda39cc0be7b56bfe59b9 3 months ago 910 B + +# debian latest f6fab3b798be 10 weeks ago 85.1 MB +# debian latest f6fab3b798be3174f45aa1eb731f8182705555f89c9026d8c1ef230cbf8301dd 10 weeks ago 85.1 MB + +if ! command -v curl &> /dev/null; then + echo >&2 'error: "curl" not found!' + exit 1 +fi + +usage() { + echo "usage: $0 dir image[:tag][@digest] ..." + echo " $0 /tmp/old-hello-world hello-world:latest@sha256:8be990ef2aeb16dbcb9271ddfe2610fa6658d13f6dfb8bc72074cc1ca36966a7" + [ -z "$1" ] || exit "$1" +} + +dir="$1" # dir for building tar in +shift || usage 1 >&2 + +[ $# -gt 0 -a "$dir" ] || usage 2 >&2 +mkdir -p "$dir" + +# hacky workarounds for Bash 3 support (no associative arrays) +images=() +rm -f "$dir"/tags-*.tmp +# repositories[busybox]='"latest": "...", "ubuntu-14.04": "..."' + +while [ $# -gt 0 ]; do + imageTag="$1" + shift + image="${imageTag%%[:@]*}" + imageTag="${imageTag#*:}" + digest="${imageTag##*@}" + tag="${imageTag%%@*}" + + # add prefix library if passed official image + if [[ "$image" != *"/"* ]]; then + image="library/$image" + fi + + imageFile="${image//\//_}" # "/" can't be in filenames :) + + token="$(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output .token)" + + manifestJson="$(curl -sSL -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$image/manifests/$digest")" + if [ "${manifestJson:0:1}" != '{' ]; then + echo >&2 "error: /v2/$image/manifests/$digest returned something unexpected:" + echo >&2 " $manifestJson" + exit 1 + fi + + layersFs=$(echo "$manifestJson" | jq --raw-output '.fsLayers | .[] | .blobSum') + + IFS=$'\n' + layers=( ${layersFs} ) + unset IFS + + history=$(echo "$manifestJson" | jq '.history | [.[] | .v1Compatibility]') + imageId=$(echo "$history" | jq --raw-output .[0] | jq --raw-output .id) + + if [ -s "$dir/tags-$imageFile.tmp" ]; then + echo -n ', ' >> "$dir/tags-$imageFile.tmp" + else + images=( "${images[@]}" "$image" ) + fi + echo -n '"'"$tag"'": "'"$imageId"'"' >> "$dir/tags-$imageFile.tmp" + + echo "Downloading '${image}:${tag}@${digest}' (${#layers[@]} layers)..." + for i in "${!layers[@]}"; do + imageJson=$(echo "$history" | jq --raw-output .[${i}]) + imageId=$(echo "$imageJson" | jq --raw-output .id) + imageLayer=${layers[$i]} + + mkdir -p "$dir/$imageId" + echo '1.0' > "$dir/$imageId/VERSION" + + echo "$imageJson" > "$dir/$imageId/json" + + # TODO figure out why "-C -" doesn't work here + # "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume." + # "HTTP/1.1 416 Requested Range Not Satisfiable" + if [ -f "$dir/$imageId/layer.tar" ]; then + # TODO hackpatch for no -C support :'( + echo "skipping existing ${imageId:0:12}" + continue + fi + curl -SL --progress -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$image/blobs/$imageLayer" -o "$dir/$imageId/layer.tar" # -C - + done + echo +done + +echo -n '{' > "$dir/repositories" +firstImage=1 +for image in "${images[@]}"; do + imageFile="${image//\//_}" # "/" can't be in filenames :) + image="${image#library\/}" + + [ "$firstImage" ] || echo -n ',' >> "$dir/repositories" + firstImage= + echo -n $'\n\t' >> "$dir/repositories" + echo -n '"'"$image"'": { '"$(cat "$dir/tags-$imageFile.tmp")"' }' >> "$dir/repositories" +done +echo -n $'\n}\n' >> "$dir/repositories" + +rm -f "$dir"/tags-*.tmp + +echo "Download of images into '$dir' complete." +echo "Use something like the following to load the result into a Docker daemon:" +echo " tar -cC '$dir' . | docker load" diff --git a/hack/make/.ensure-frozen-images b/hack/make/.ensure-frozen-images index 16458741c970c..12603c70ba66c 100644 --- a/hack/make/.ensure-frozen-images +++ b/hack/make/.ensure-frozen-images @@ -4,17 +4,17 @@ set -e # this list should match roughly what's in the Dockerfile (minus the explicit image IDs, of course) images=( busybox:latest - hello-world:frozen + hello-world:latest jess/unshare:latest ) # on ARM we need images that work for the ARM architecture -if [ -v DOCKER_ENGINE_OSARCH ] && [ "$DOCKER_ENGINE_OSARCH" = "linux/arm" ]; then - images=( - hypriot/armhf-busybox@ea0800bb83571c585c5652b53668e76b29c7c0eef719892f9d0a48607984f9e1 - hypriot/armhf-hello-world@508c59a4f8b23c77bbcf43296c3f580873dc7eecb1f0d680cea3067e221fd4c2 - hypriot/armhf-unshare@3f1db65f8bbabc743fd739cf7145a56c35b2a0979ae3174e9d79b7fa4b00fca1 - ) +if [ "$DOCKER_ENGINE_OSARCH" = "linux/arm" ]; then + images=( + hypriot/armhf-busybox@ea0800bb83571c585c5652b53668e76b29c7c0eef719892f9d0a48607984f9e1 + hypriot/armhf-hello-world@508c59a4f8b23c77bbcf43296c3f580873dc7eecb1f0d680cea3067e221fd4c2 + hypriot/armhf-unshare@3f1db65f8bbabc743fd739cf7145a56c35b2a0979ae3174e9d79b7fa4b00fca1 + ) fi if ! docker inspect "${images[@]}" &> /dev/null; then @@ -23,10 +23,10 @@ if ! docker inspect "${images[@]}" &> /dev/null; then ( set -x; tar -cC "$hardCodedDir" . | docker load ) else dir="$DEST/frozen-images" - # extract the exact "RUN download-frozen-image.sh" line from the Dockerfile itself for consistency - # NOTE: this will fail if either "curl" is not installed or if the Dockerfile is not available/readable + # extract the exact "RUN download-frozen-image-v2.sh" line from the Dockerfile itself for consistency + # NOTE: this will fail if either "curl" or "jq" is not installed or if the Dockerfile is not available/readable awk ' - $1 == "RUN" && $2 == "./contrib/download-frozen-image.sh" { + $1 == "RUN" && $2 == "./contrib/download-frozen-image-v2.sh" { for (i = 2; i < NF; i++) printf ( $i == "'"$hardCodedDir"'" ? "'"$dir"'" : $i ) " "; print $NF; @@ -46,14 +46,16 @@ if ! docker inspect "${images[@]}" &> /dev/null; then fi fi -if [ -v DOCKER_ENGINE_OSARCH ] && [ "$DOCKER_ENGINE_OSARCH" = "linux/arm" ]; then - # tag images to ensure that all integrations work with the defined image names - docker tag hypriot/armhf-busybox:latest busybox:latest - docker tag hypriot/armhf-hello-world:latest hello-world:frozen - docker tag hypriot/armhf-unshare:latest jess/unshare:latest +if [ "$DOCKER_ENGINE_OSARCH" = "linux/arm" ]; then + # tag images to ensure that all integrations work with the defined image names + docker tag hypriot/armhf-busybox:latest busybox:latest + docker tag hypriot/armhf-hello-world:latest hello-world:frozen + docker tag hypriot/armhf-unshare:latest jess/unshare:latest - # remove orignal tags as these make problems with later tests: TestInspectApiImageResponse - docker rmi hypriot/armhf-busybox:latest - docker rmi hypriot/armhf-hello-world:latest - docker rmi hypriot/armhf-unshare:latest + # remove orignal tags as these make problems with later tests: TestInspectApiImageResponse + docker rmi hypriot/armhf-busybox:latest + docker rmi hypriot/armhf-hello-world:latest + docker rmi hypriot/armhf-unshare:latest +else + docker tag hello-world:latest hello-world:frozen fi diff --git a/integration-cli/docker_cli_run_test.go b/integration-cli/docker_cli_run_test.go index 546ad4d5167f2..b839ee78cfd28 100644 --- a/integration-cli/docker_cli_run_test.go +++ b/integration-cli/docker_cli_run_test.go @@ -2868,7 +2868,7 @@ func (s *DockerSuite) TestRunUnshareProc(c *check.C) { /* Ensure still fails if running privileged with the default policy */ name = "crashoverride" - if out, _, err := dockerCmdWithError("run", "--privileged", "--security-opt", "apparmor:docker-default", "--name", name, "jess/unshare", "unshare", "-p", "-m", "-f", "-r", "mount", "-t", "proc", "none", "/proc"); err == nil || !(strings.Contains(out, "Permission denied") || strings.Contains(out, "Operation not permitted")) { + if out, _, err := dockerCmdWithError("run", "--privileged", "--security-opt", "apparmor:docker-default", "--name", name, "jess/unshare", "unshare", "-p", "-m", "-f", "-r", "mount", "-t", "proc", "none", "/proc"); err == nil || !(strings.Contains(strings.ToLower(out), "permission denied") || strings.Contains(strings.ToLower(out), "operation not permitted")) { c.Fatalf("unshare should have failed with permission denied, got: %s, %v", out, err) } }