Fast pulls and pushes (#71)

* This implements fast docker_{push,pull}

This commit pulls in the latest release of `google/containerregistry`, which includes a new fast-path for puller/pusher to use, which is consistent with our new intermediate format.

This commit adapts `docker_pull` to consume the new puller and stitch together the image's constituent elements using `docker_import`.

This commit also changes `docker_push` to consume the new options on the new pusher to publish images without linking the full `docker save` tarball, which can be quite expensive for large images.

* Fix the layer ordering from docker_pull

* Remove the reversed layer ordering after picking up the fix in our diffbase.

* Remove another unused utility.

* Update the documentation to drop :image.tar everywhere

README.md

@@ -174,7 +174,7 @@ These variables are specified on the CLI using:
 docker_build(
     name = "app",
     # References docker_pull from WORKSPACE (above)
-    base = "@java_base//image:image.tar",
+    base = "@java_base//image",
     files = ["//java/com/example/app:Hello_deploy.jar"],
     cmd = ["Hello_deploy.jar"]
 )
@@ -208,7 +208,7 @@ docker_pull(
 )
 ```
 
-This can then be referenced in `BUILD` files as `@base//image:image.tar`.
+This can then be referenced in `BUILD` files as `@base//image`.
 
 ### docker_push
 
@@ -237,7 +237,7 @@ docker_pull(
 )
 ```
 
-This can then be referenced in `BUILD` files as `@official_ubuntu//image:image.tar`.
+This can then be referenced in `BUILD` files as `@official_ubuntu//image`.
 
 ### docker_pull (Quay.io)
 
@@ -252,7 +252,7 @@ docker_pull(
 )
 ```
 
-This can then be referenced in `BUILD` files as `@etcd//image:image.tar`.
+This can then be referenced in `BUILD` files as `@etcd//image`.
 
 ### docker_pull (Bintray.io)
 
@@ -266,7 +266,7 @@ docker_pull(
 )
 ```
 
-This can then be referenced in `BUILD` files as `@artifactory//image:image.tar`.
+This can then be referenced in `BUILD` files as `@artifactory//image`.
 
 ### docker_pull (Gitlab)
 
@@ -281,7 +281,7 @@ docker_pull(
 )
 ```
 
-This can then be referenced in `BUILD` files as `@gitlab//image:image.tar`.
+This can then be referenced in `BUILD` files as `@gitlab//image`.
 
 **NOTE:** This will only work on systems with Python >2.7.6
 

docker/BUILD

@@ -60,8 +60,8 @@ TEST_DATA = [
     "//docker/testdata:stamped_bundle_test",
     "//docker/testdata:pause_based.tar",
     "//docker/testdata:dashdash_entrypoint.tar",
-    "//docker/testdata:base_based.tar",
-    "//docker/testdata:cc_based.tar",
+    "//docker/testdata:base_based_warning.tar",
+    "//docker/testdata:cc_based_warning.tar",
     "//docker/testdata:pause_piecemeal.tar",
 ]
 

docker/build.bzl

@@ -39,7 +39,6 @@ load(
     _incr_load = "incremental_load",
     _layer_tools = "tools",
 )
-load(":list.bzl", "reverse")
 load(
     ":path.bzl",
     "dirname",
@@ -205,10 +204,6 @@ def _docker_build_impl(ctx):
       # A list of paths to the layer digests.
       "blobsum": shas,
 
-      # TODO(mattmoor): docker_pull is going to need to produce all of
-      # these files, so figure out if/how we're going to produce them
-      # efficiently before getting too happy with this solution.
-
       # A list of paths to the layer .tar files
       "unzipped_layer": unzipped_layers,
       # A list of paths to the layer diff_ids.

docker/bundle.bzl

@@ -24,7 +24,6 @@ load(
     _incr_load = "incremental_load",
     _layer_tools = "tools",
 )
-load(":list.bzl", "reverse")
 
 def _docker_bundle_impl(ctx):
   """Implementation for the docker_bundle rule."""

docker/docker.bzl

@@ -28,23 +28,23 @@ load(":pull.bzl", "docker_pull")
 load(":push.bzl", "docker_push")
 
 # The release of the github.com/google/containerregistry to consume.
-CONTAINERREGISTRY_RELEASE = "v0.0.11"
+CONTAINERREGISTRY_RELEASE = "v0.0.12"
 
 def docker_repositories():
   """Download dependencies of docker rules."""
   native.http_file(
     name = "puller",
     url = ("https://storage.googleapis.com/containerregistry-releases/" +
            CONTAINERREGISTRY_RELEASE + "/puller.par"),
-    sha256 = "90a76d01ee57a5df7353a533c96966822c9efda61636c11eac11344171556017",
+    sha256 = "7ebdefbb8a98c5bf8b737a7891a8860ed2f22ae149d3dba200bc104305b6e1d6",
     executable = True,
   )
 
   native.http_file(
     name = "pusher",
     url = ("https://storage.googleapis.com/containerregistry-releases/" +
            CONTAINERREGISTRY_RELEASE + "/pusher.par"),
-    sha256 = "1989ceb41144784dccb3476cea5b5f1bef112bb0aae56544b9c56572894c38ab",
+    sha256 = "a8e48dd99b3e85ac5fe30e1daec327ee509540ff18f5ad7f4724d84ccfd3d150",
     executable = True,
   )
 

docker/layers.bzl

@@ -13,7 +13,6 @@
 # limitations under the License.
 """Tools for dealing with Docker Image layers."""
 
-load(":list.bzl", "reverse")
 load(
     ":path.bzl",
     _get_runfile_path = "runfile",

docker/pull.bzl

@@ -24,16 +24,21 @@ def _impl(repository_ctx):
   # Add an empty top-level BUILD file.
   repository_ctx.file("BUILD", "")
 
-  # TODO(mattmoor): Is there a way of doing this so that
-  # consumers can just depend on @base//image ?
   repository_ctx.file("image/BUILD", """
 package(default_visibility = ["//visibility:public"])
-exports_files(["image.tar"])
+
+load("@io_bazel_rules_docker//docker:import.bzl", "docker_import")
+
+docker_import(
+  name = "image",
+  config = "config.json",
+  layers = glob(["*.tar.gz"]),
+)
 """)
 
   args = [
       repository_ctx.path(repository_ctx.attr._puller),
-      "--tarball", repository_ctx.path("image/image.tar")
+      "--directory", repository_ctx.path("image")
   ]
 
   # If a digest is specified, then pull by digest.  Otherwise, pull by tag.

docker/push-tag.sh.tpl

@@ -15,4 +15,4 @@
 
 %{docker_pusher} \
   --name=%{registry}/%{repository}:%{tag} \
-  %{stamp} --tarball=%{image} "$@"
+  %{stamp} %{image} "$@"

docker/push.bzl

@@ -21,15 +21,38 @@ load(
     ":path.bzl",
     _get_runfile_path = "runfile",
 )
+load(
+    ":layers.bzl",
+    _get_layers = "get_from_target",
+    _layer_tools = "tools"
+)
 
 def _impl(ctx):
   """Core implementation of docker_push."""
   stamp_inputs = []
   if ctx.attr.stamp:
     stamp_inputs = [ctx.info_file, ctx.version_file]
 
+  image = _get_layers(ctx, ctx.attr.image, ctx.files.image)
+
   stamp_arg = " ".join(["--stamp-info-file=%s" % f.short_path for f in stamp_inputs])
 
+  # Leverage our efficient intermediate representation to push.
+  legacy_base_arg = ""
+  if image.get("legacy"):
+    print("Pushing an image based on a tarball can be very " +
+          "expensive.  If the image is the output of a " +
+          "docker_build, consider dropping the '.tar' extension. " +
+          "If the image is checked in, consider using " +
+          "docker_import instead.")
+    legacy_base_arg = "--tarball=%s" % image["legacy"].short_path
+
+  blobsums = image.get("blobsum", [])
+  digest_arg = " ".join(["--digest=%s" % f.short_path for f in blobsums])
+  blobs = image.get("zipped_layer", [])
+  layer_arg = " ".join(["--layer=%s" % f.short_path for f in blobs])
+  config_arg = "--config=%s" % image["config"].short_path
+
   ctx.template_action(
       template = ctx.file._tag_tpl,
       substitutions = {
@@ -40,17 +63,19 @@ def _impl(ctx):
           "%{stamp}": stamp_arg,
           "%{tag}": ctx.expand_make_variables(
               "tag", ctx.attr.tag, {}),
-          "%{image}": ctx.file.image.short_path,
+          "%{image}": "%s %s %s %s" % (
+              legacy_base_arg, config_arg, digest_arg, layer_arg),
           "%{docker_pusher}": ctx.executable._pusher.short_path,
       },
       output = ctx.outputs.executable,
       executable=True,
   )
 
   return struct(runfiles = ctx.runfiles(files = [
-      ctx.file.image,
-      ctx.executable._pusher
-  ] + stamp_inputs))
+      ctx.executable._pusher,
+      image["config"]
+  ] + image.get("blobsum", []) + image.get("zipped_layer", []) +
+  stamp_inputs +  ([image["legacy"]] if image.get("legacy") else [])))
 
 _docker_push = rule(
     attrs = {
@@ -77,7 +102,7 @@ _docker_push = rule(
             default = False,
             mandatory = False,
         ),
-    },
+    } + _layer_tools,
     executable = True,
     implementation = _impl,
 )
@@ -94,8 +119,4 @@ def docker_push(image=None, **kwargs):
     repository: the name of the image.
     tag: (optional) the tag of the image, default to 'latest'.
   """
-
-  if not image.endswith(".tar"):
-    image = image + ".tar"
-
   _docker_push(image=image, **kwargs)

docker/testdata/BUILD

@@ -267,6 +267,24 @@ docker_build(
     files = ["foo"],
 )
 
+# We should get a warning when we build/run this.
+docker_push(
+    name = "push_inadvertent_legacy_base",
+    image = ":with_env.tar",
+    registry = "gcr.io",
+    repository = "convoy-adapter/bazel-test",
+    tag = "with_env",
+)
+
+# We should get a warning when we build/run this.
+docker_push(
+    name = "push_legacy_base",
+    image = ":pause_based",
+    registry = "gcr.io",
+    repository = "convoy-adapter/bazel-test",
+    tag = "pause_based",
+)
+
 docker_bundle(
     name = "bundle_test",
     images = {
@@ -338,15 +356,21 @@ docker_build(
 )
 
 docker_build(
-    name = "base_based",
+    name = "base_based_warning",
     base = "@distroless_base//image:image.tar",
 )
 
 docker_build(
-    name = "cc_based",
+    name = "cc_based_warning",
     base = "@distroless_cc//image:image.tar",
 )
 
+# Tests loading a docker_pull'd base.
+docker_build(
+    name = "cc_based",
+    base = "@distroless_cc//image",
+)
+
 PAUSE_CONFIG = "2b58359142b094165abfc9ad9c327c453a3eaa624d0c5ee8f0ed1ab50483603b.json"
 
 genrule(
@@ -374,5 +398,5 @@ PAUSE_LAYERS = [
 docker_import(
    name = "pause_piecemeal",
    config = ":pause_config.json",
-    layers = ["%s.tar.gz" % x for x in PAUSE_LAYERS],
+   layers = ["%s.tar.gz" % x for x in PAUSE_LAYERS],
 )

testing/e2e.sh

@@ -32,7 +32,7 @@ load(
 
 docker_build(
   name = "pause_based",
-  base = "@pause//image:image.tar",
+  base = "@pause//image",
   workdir = "/tmp",
 )
 EOF