From 4483612588f0591a0ade292a99761f2cc5198761 Mon Sep 17 00:00:00 2001
From: Utku Demir <me@utdemir.com>
Date: Thu, 28 May 2020 14:55:42 +1200
Subject: [PATCH] Calculate sha256 on the fly on dockerTools.buildLayeredImage

Previously, we were calculating layer checksums at a separate pass after
the creation of the tarball. This is not ideal when packaging containers
with large layers, since it requires reading the whole tarball again
from disk in order to create the checksum.

This commit calculates the sha256 in parallel when creating the tarball
and saves it to a file for later use by buildLayeredImage function.
---
 pkgs/build-support/docker/default.nix            | 14 +++++++++++---
 pkgs/build-support/docker/store-path-to-layer.sh |  1 +
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix
index 83f4a9e0c01b59..9571d3ac5cbfc5 100644
--- a/pkgs/build-support/docker/default.nix
+++ b/pkgs/build-support/docker/default.nix
@@ -395,7 +395,13 @@ rec {
       # Tar up the layer and throw it into 'layer.tar', while calculating its checksum.
       echo "Packing layer..."
       mkdir $out
-      tarhash=$(tar --transform='s|^\./||' -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf - . | tee $out/layer.tar | tarsum)
+      tarhash=$(
+        tar --transform='s|^\./||' -C layer \
+            --sort=name --mtime="@$SOURCE_DATE_EPOCH" \
+            --owner=${toString uid} --group=${toString gid} -cf - . |
+          tee $out/layer.tar |
+          tee >(sha256sum | cut -d ' ' -f 1 > "$out/sha256") |
+          tarsum)
 
       # Add a 'checksum' field to the JSON, with the value set to the
       # checksum of the tarball.
@@ -626,9 +632,11 @@ rec {
         imageJson=$(cat ${configJson} | jq ". + {\"rootfs\": {\"diff_ids\": [], \"type\": \"layers\"}}")
         manifestJson=$(jq -n "[{\"RepoTags\":[\"$imageName:$imageTag\"]}]")
         for layer in $(cat layer-list); do
-          layerChecksum=$(sha256sum $layer/layer.tar | cut -d ' ' -f1)
+          layerChecksum=$(cat $layer/sha256)
           layerID=$(sha256sum "$layer/json" | cut -d ' ' -f 1)
-          ln -s "$layer" "./image/$layerID"
+
+          mkdir "./image/$layerID"
+          ln -s -t "./image/$layerID" "$layer/"{layer.tar,json,VERSION}
 
           manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= . + [\"$layerID/layer.tar\"]")
           imageJson=$(echo "$imageJson" | jq ".history |= . + [{\"created\": \"$(jq -r .created ${configJson})\"}]")
diff --git a/pkgs/build-support/docker/store-path-to-layer.sh b/pkgs/build-support/docker/store-path-to-layer.sh
index 7437da51cc4a45..4a49701ba94f14 100755
--- a/pkgs/build-support/docker/store-path-to-layer.sh
+++ b/pkgs/build-support/docker/store-path-to-layer.sh
@@ -39,6 +39,7 @@ tarhash=$(
       --transform 's,^nix/store$,/\0,' \
       --transform 's,^[^/],/nix/store/\0,rS' |
     tee "$layerPath/layer.tar" |
+    tee >(sha256sum | cut -d ' ' -f 1 > $layerPath/sha256) |
     tarsum
 )