From 15524c41b27697c478981cb2df7d5d7df02f3ba4 Mon Sep 17 00:00:00 2001
From: Marcelo Vanzin <vanzin@cloudera.com>
Date: Thu, 18 Oct 2018 10:21:37 -0700
Subject: [PATCH] [SPARK-25682][K8S] Package example jars in same target for
 dev and distro images.

This way the image generated from both environments has the same layout,
with just a difference in contents that should not affect functionality.

Also added some minor error checking to the image script.

Closes #22681 from vanzin/SPARK-25682.

Authored-by: Marcelo Vanzin <vanzin@cloudera.com>
Signed-off-by: Marcelo Vanzin <vanzin@cloudera.com>
---
 bin/docker-image-tool.sh                         | 16 ++++++++++++++++
 .../docker/src/main/dockerfiles/spark/Dockerfile |  5 ++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/bin/docker-image-tool.sh b/bin/docker-image-tool.sh
index 228494de6d5a1..f17791a2a2d86 100755
--- a/bin/docker-image-tool.sh
+++ b/bin/docker-image-tool.sh
@@ -47,6 +47,11 @@ function build {
 
   if [ ! -f "$SPARK_HOME/RELEASE" ]; then
     # Set image build arguments accordingly if this is a source repo and not a distribution archive.
+    #
+    # Note that this will copy all of the example jars directory into the image, and that will
+    # contain a lot of duplicated jars with the main Spark directory. In a proper distribution,
+    # the examples directory is cleaned up before generating the distribution tarball, so this
+    # issue does not occur.
     IMG_PATH=resource-managers/kubernetes/docker/src/main/dockerfiles
     BUILD_ARGS=(
       ${BUILD_PARAMS}
@@ -55,6 +60,8 @@ function build {
       --build-arg
       spark_jars=assembly/target/scala-$SPARK_SCALA_VERSION/jars
       --build-arg
+      example_jars=examples/target/scala-$SPARK_SCALA_VERSION/jars
+      --build-arg
       k8s_tests=resource-managers/kubernetes/integration-tests/tests
     )
   else
@@ -78,14 +85,23 @@ function build {
   docker build $NOCACHEARG "${BUILD_ARGS[@]}" \
     -t $(image_ref spark) \
     -f "$BASEDOCKERFILE" .
+  if [[ $? != 0 ]]; then
+    error "Failed to build Spark docker image."
+  fi
 
   docker build $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" \
     -t $(image_ref spark-py) \
     -f "$PYDOCKERFILE" .
+  if [[ $? != 0 ]]; then
+    error "Failed to build PySpark docker image."
+  fi
 
   docker build $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" \
     -t $(image_ref spark-r) \
     -f "$RDOCKERFILE" .
+  if [[ $? != 0 ]]; then
+    error "Failed to build SparkR docker image."
+  fi
 }
 
 function push {
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
index 4bada0d753215..5f469c30a96fa 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
@@ -18,6 +18,7 @@
 FROM openjdk:8-alpine
 
 ARG spark_jars=jars
+ARG example_jars=examples/jars
 ARG img_path=kubernetes/dockerfiles
 ARG k8s_tests=kubernetes/tests
 
@@ -32,6 +33,7 @@ RUN set -ex && \
     apk upgrade --no-cache && \
     apk add --no-cache bash tini libc6-compat linux-pam krb5 krb5-libs && \
     mkdir -p /opt/spark && \
+    mkdir -p /opt/spark/examples && \
     mkdir -p /opt/spark/work-dir && \
     touch /opt/spark/RELEASE && \
     rm /bin/sh && \
@@ -43,7 +45,8 @@ COPY ${spark_jars} /opt/spark/jars
 COPY bin /opt/spark/bin
 COPY sbin /opt/spark/sbin
 COPY ${img_path}/spark/entrypoint.sh /opt/
-COPY examples /opt/spark/examples
+COPY ${example_jars} /opt/spark/examples/jars
+COPY examples/src /opt/spark/examples/src
 COPY ${k8s_tests} /opt/spark/tests
 COPY data /opt/spark/data