perf(tools): fabric 1.x AIO image pre-fetching #649

Fixes #649

In addition to that fix it also fixes a pre-fetching bug in the 2.x
image which was stopping it from actually using the pre-fetched images
on the file-system => the directory paths had to be specified
one by one for each image that we pre-fetched, it did not
work if you specified a common parent directory where the
images were stored, so this commit provides a fix for both
1.4.x and 2.x in the sense that they both should no longer
hit DockerHub for image downloads at container startup
time, only build-time as long as the FABRIC_VERSION
env vars were not altered by the person launching the
containers (if they were then those different versions
will still have to be fetched at runtime).

The biggest net positive effect(s) of this commit:
1. Image boot will be much faster, especially on machines
where the network is slow.
2. The risk of getting slapped with DockerHub image pull
rate limiting goes down significantly.

Also fixes a health check bug where the FABRIC_VERSION build arg
was not being exposed as an ENV var
in the container and the parsing of the
version was failing due to this.

Signed-off-by: Peter Somogyvari <peter.somogyvari@accenture.com>

tools/docker/fabric-all-in-one/Dockerfile_v1.4.x

@@ -133,11 +133,50 @@ RUN apk add --no-cache util-linux
 # FIXME - make it so that SSHd does not need this to work
 RUN echo "root:$(uuidgen)" | chpasswd
 
+RUN curl -sSL https://raw.githubusercontent.com/cloudflare/semver_bash/c1133faf0efe17767b654b213f212c326df73fa3/semver.sh > /semver.sh
+RUN chmod +x /semver.sh
+
+# jq is needed by the /download-frozen-image-v2.sh script to pre-fetch docker images without docker.
+RUN apk add --no-cache jq
+
+# Get the utility script that can pre-fetch the Fabric docker images without
+# a functioning Docker daemon available which we do not have at image build
+# time so have to resort to manually get the Fabric images insteadd of just saying
+# "docker pull hyperledger/fabric..." etc.
+# The reason to jump trough these hoops is to speed up the boot time of the
+# container which won't have to download the images at container startup since
+# they'll have been cached already at build time.
+RUN curl -sSL https://raw.githubusercontent.com/moby/moby/dedf8528a51c6db40686ed6676e9486d1ed5f9c0/contrib/download-frozen-image-v2.sh > /download-frozen-image-v2.sh
+RUN chmod +x /download-frozen-image-v2.sh
+
+RUN mkdir -p /etc/hyperledger/fabric/fabric-peer/
+RUN mkdir -p /etc/hyperledger/fabric/fabric-orderer/
+RUN mkdir -p /etc/hyperledger/fabric/fabric-ccenv/
+RUN mkdir -p /etc/hyperledger/fabric/fabric-tools/
+RUN mkdir -p /etc/hyperledger/fabric/fabric-ca/
+
+RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/fabric-peer/ hyperledger/fabric-peer:${FABRIC_VERSION}
+RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/fabric-orderer/ hyperledger/fabric-orderer:${FABRIC_VERSION}
+RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/fabric-ccenv/ hyperledger/fabric-ccenv:${FABRIC_VERSION}
+RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/fabric-tools/ hyperledger/fabric-tools:${FABRIC_VERSION}
+RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/fabric-ca/ hyperledger/fabric-ca:${CA_VERSION}
+
+# Install supervisord because we need to run the docker daemon and also the fabric network
+# meaning that we have multiple processes to run.
+RUN apk add --no-cache supervisor
+
+COPY supervisord.conf /etc/supervisord.conf
+COPY run-fabric-network.sh /
+COPY healthcheck.sh /
+
+ENV FABRIC_VERSION=${FABRIC_VERSION}
+ENV CA_VERSION=${CA_VERSION}
+
 # Extend the parent image's entrypoint
 # https://superuser.com/questions/1459466/can-i-add-an-additional-docker-entrypoint-script
 ENTRYPOINT ["/usr/bin/supervisord"]
 CMD ["--configuration", "/etc/supervisord.conf", "--nodaemon"]
 
 # We consider the container healthy once the default example fabcar contract has been deployed
 # and is responsive to queries as well
-HEALTHCHECK --interval=1s --timeout=5s --start-period=60s --retries=300 CMD ./healthcheck.sh
+HEALTHCHECK --interval=5s --timeout=5s --start-period=30s --retries=300 CMD ./healthcheck.sh

tools/docker/fabric-all-in-one/Dockerfile_v2.x

@@ -131,13 +131,19 @@ RUN apk add --no-cache jq
 RUN curl -sSL https://raw.githubusercontent.com/moby/moby/dedf8528a51c6db40686ed6676e9486d1ed5f9c0/contrib/download-frozen-image-v2.sh > /download-frozen-image-v2.sh
 RUN chmod +x /download-frozen-image-v2.sh
 
-RUN mkdir -p /etc/hyperledger/fabric
-RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/ hyperledger/fabric-peer:${FABRIC_VERSION}
-RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/ hyperledger/fabric-orderer:${FABRIC_VERSION}
-RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/ hyperledger/fabric-ccenv:${FABRIC_VERSION}
-RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/ hyperledger/fabric-tools:${FABRIC_VERSION}
-RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/ hyperledger/fabric-baseos:${FABRIC_VERSION}
-RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/ hyperledger/fabric-ca:${CA_VERSION}
+RUN mkdir -p /etc/hyperledger/fabric/fabric-peer/
+RUN mkdir -p /etc/hyperledger/fabric/fabric-orderer/
+RUN mkdir -p /etc/hyperledger/fabric/fabric-ccenv/
+RUN mkdir -p /etc/hyperledger/fabric/fabric-tools/
+RUN mkdir -p /etc/hyperledger/fabric/fabric-baseos/
+RUN mkdir -p /etc/hyperledger/fabric/fabric-ca/
+
+RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/fabric-peer/ hyperledger/fabric-peer:${FABRIC_VERSION}
+RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/fabric-orderer/ hyperledger/fabric-orderer:${FABRIC_VERSION}
+RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/fabric-ccenv/ hyperledger/fabric-ccenv:${FABRIC_VERSION}
+RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/fabric-tools/ hyperledger/fabric-tools:${FABRIC_VERSION}
+RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/fabric-baseos/ hyperledger/fabric-baseos:${FABRIC_VERSION}
+RUN /download-frozen-image-v2.sh /etc/hyperledger/fabric/fabric-ca/ hyperledger/fabric-ca:${CA_VERSION}
 
 # Download and execute the Fabric bootstrap script, but instruct it with the -d
 # flag to avoid pulling docker images because during the build phase of this image
@@ -163,6 +169,8 @@ ENV CORE_PEER_TLS_ROOTCERT_FILE=/fabric-samples/test-network/organizations/peerO
 ENV CORE_PEER_MSPCONFIGPATH=/fabric-samples/test-network/organizations/peerOrganizations/org1.example.com/users/Admin@org1.example.com/msp
 ENV CORE_PEER_ADDRESS=localhost:7051
 ENV COMPOSE_PROJECT_NAME=cactusfabrictestnetwork
+ENV FABRIC_VERSION=${FABRIC_VERSION}
+ENV CA_VERSION=${CA_VERSION}
 
 # Extend the parent image's entrypoint
 # https://superuser.com/questions/1459466/can-i-add-an-additional-docker-entrypoint-script

tools/docker/fabric-all-in-one/run-fabric-network.sh

@@ -16,14 +16,24 @@ function main()
   local SPECIAL=""
   semverParseInto "${FABRIC_VERSION}" MAJOR MINOR PATCH SPECIAL
 
-  tar -cC '/etc/hyperledger/fabric/' . | docker load
+  tar -cC '/etc/hyperledger/fabric/fabric-peer/' . | docker load
+  tar -cC '/etc/hyperledger/fabric/fabric-orderer/' . | docker load
+  tar -cC '/etc/hyperledger/fabric/fabric-ccenv/' . | docker load
+  tar -cC '/etc/hyperledger/fabric/fabric-tools/' . | docker load
+  tar -cC '/etc/hyperledger/fabric/fabric-ca/' . | docker load
 
-  /bootstrap.sh ${FABRIC_VERSION} ${CA_VERSION} -b -s
 
   echo "[FabricAIO] >>> Parsed MAJOR version of Fabric as ${MAJOR}"
 
   if [ "$MAJOR" -gt 1 ]; then
     # Major version is 2 or newer (we'll deal with 3.x when it is released)
+
+    # Fabric 2.x has this new image called fabric-baseos so we need to load that
+    # as well when we detect that we are running Fabrix 2.x not 1.x
+    tar -cC '/etc/hyperledger/fabric/fabric-baseos/' . | docker load
+
+    /bootstrap.sh ${FABRIC_VERSION} ${CA_VERSION} -b -s
+
     cd /fabric-samples/test-network/
     echo "[FabricAIO] >>> pulling up test network..."
     ./network.sh up -ca
@@ -34,6 +44,7 @@ function main()
     echo "[FabricAIO] >>> contract deployed OK."
     echo "[FabricAIO] >>> container healthcheck should begin passing in about 5-15 seconds..."
   else
+    /bootstrap.sh ${FABRIC_VERSION} ${CA_VERSION} -b -s
     # Major version is 1.x or earlier (assumption is 1.4.x only)
     cd /fabric-samples/fabcar/
     ./startFabric.sh