YETUS-791. Add support for --cache-from to speed up building

Signed-off-by: Allen Wittenauer <aw@apache.org>

Jenkinsfile

@@ -172,8 +172,10 @@ pipeline {
                 # run in docker mode and specifically point to our
                 # Dockerfile since we don't want to use the auto-pulled version.
                 if [[ "${USE_DOCKER_FLAG}" == true ]]; then
+                  docker pull ubuntu:xenial
                   YETUS_ARGS+=("--docker")
                   YETUS_ARGS+=("--dockerfile=${YETUS_DOCKERFILE}")
+                  YETUS_ARGS+=("--docker-cache-from=apache/yetus-base:master")
                 else
                   # need to figure this out programmatically; hard-coded for now
                   export JAVA_HOME=/home/jenkins/tools/java/latest1.8

asf-site-src/source/documentation/in-progress/precommit-docker.md

@@ -57,6 +57,10 @@ In order to use both 'YETUS CUT HERE' and a Dockerfile that uses COPY and ADD di
 
 Instead of processing a Dockerfile, test-patch can pull a tag from a repository using the `--docker-tag` parameter. Note that the repository must already be logged into and configured prior to executing test-patch.
 
+## Using a cache
+
+With the `--docker-cache-from` parameter, other images may be utilized to provide a cache when building a Dockerfile. This comma delimited list will automatically be pulled (errors are ignored) and given to the docker command line to use.
+
 ## Platforms
 
 When either building or pull an image, test-patch supports the `--docker-platform` flag to pass in the Docker `--platform` flag.  This allows you full control over what kind of image the software either creates or fetches.

precommit/src/main/shell/core.d/docker.sh

@@ -62,6 +62,7 @@ function docker_usage
   yetus_add_option "--dockercmd=<file>" "Command to use as docker executable (default: '${DOCKERCMD}')"
   if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then
     yetus_add_option "--docker-bash-debug=<bool>" "Enable bash -x mode running in a container (default: ${YETUS_DOCKER_BASH_DEBUG})"
+    yetus_add_option "--docker-cache-from=<image>" "Comma delimited images to use as a cache when building"
     yetus_add_option "--dockerfile=<file>" "Dockerfile fragment to use as the base (default: '${DOCKERFILE_DEFAULT}')"
     yetus_add_option "--dockerind=<bool>" "Enable Docker-in-Docker by mounting the Docker socket in the container (default: '${DOCKER_IN_DOCKER}')"
     yetus_add_option "--docker-platform=<plat>" "Use a platform string for building and pulling (default: ${DOCKER_PLATFORM})"
@@ -93,6 +94,9 @@ function docker_parse_args
         YETUS_DOCKER_BASH_DEBUG=${i#*=}
         add_docker_env YETUS_DOCKER_BASH_DEBUG
       ;;
+      --docker-cache-from=*)
+        DOCKER_CACHE_FROM=${i#*=}
+      ;;
       --dockercmd=*)
         #shellcheck disable=SC2034
         DOCKERCMD=${i#*=}
@@ -561,6 +565,8 @@ function docker_run_image
   declare lines
   declare dockerversion
   declare -a dockplat
+  declare -a cachefrom
+  declare -a images
 
   big_console_header "Docker Image Creation"
   start_clock
@@ -633,8 +639,17 @@ function docker_run_image
       cp -p "${buildfile}" "${PATCH_DIR}/Dockerfile"
     fi
 
+    if [[ -n "${DOCKER_CACHE_FROM}" ]]; then
+      yetus_comma_to_array images "${DOCKER_CACHE_FROM}"
+      for i in "${images[@]}"; do
+        docker pull "${i}" || true
+      done
+      cachefrom=("--cache-from=yetus/${PROJECT_NAME}:${gitfilerev},${DOCKER_CACHE_FROM}")
+    fi
+
     if ! dockercmd build \
           "${dockplat[@]}" \
+          "${cachefrom[@]}" \
           --label org.apache.yetus=\"\" \
           --label org.apache.yetus.testpatch.project="${PROJECT_NAME}" \
           --tag "${baseimagename}" \

start-build-env.sh

@@ -26,11 +26,14 @@ if [[ "${BRANCH}" =~ HEAD ]]; then
   BRANCH=$(git branch | grep '\*' | awk '{print $NF}'  | sed -e s,rel/,,g -e s,\),,g )
 fi
 
-echo "Attempting a pull of apache/yetus-base:${BRANCH} and apache/yetus-base:master to save time"
+echo "Attempting a few pulls of apache/yetus and apache/yetus-base to save time"
 echo "Errors here will be ignored!"
 docker pull "apache/yetus-base:${BRANCH}" || docker pull "apache/yetus-base:master" || true
+docker pull "apache/yetus:${BRANCH}"  || docker pull "apache/yetus:master" || true
 
-docker build -t "apache/yetus-build:${BRANCH}" .
+docker build \
+  --cache-from="apache/yetus-base:${BRANCH},apache/yetus-base:master,apache/yetus:${BRANCH},apache/yetus:master" \
+  -t "apache/yetus-build:${BRANCH}" .
 
 USER_NAME=${SUDO_USER:=$USER}
 USER_ID=$(id -u "${USER_NAME}")