ORC-766: Generalize the docker scripts to handle build arguments.

docker/os-list.txt

@@ -5,4 +5,6 @@ debian10
 ubuntu16
 ubuntu18
 ubuntu20
-ubuntu20-clang
+centos8_jdk=11
+ubuntu20_jdk=11
+ubuntu20_jdk=11_cc=clang

docker/reinit.sh

@@ -16,15 +16,17 @@
 # limitations under the License.
 
 start=`date`
-for jdk in 8 11; do
-  for os in `cat os-list.txt`; do
-    if [[ "$os" = "debian10" && "$jdk" = "8" ]] || [[ "$os" = "debian9" && "$jdk" = "11" ]] || [[ "$os" = "ubuntu16" && "$jdk" = "11" ]]; then
-      echo "Skip an initialize $os with $jdk"
-      continue
-    fi
-    echo "Re-initialize $os with $jdk"
-    ( cd $os && docker build --no-cache -t "orc-${os}-jdk${jdk}" --build-arg jdk=${jdk} . )
-  done
+for build in `cat os-list.txt`; do
+  echo "Re-initialize $build"
+  OS=$(echo "$build" | cut -d '_' -f1)
+  REST=$(echo "$build" | cut -d '_' -f2- -s)
+  if [ -z "$REST" ]; then
+    ARGS=""
+  else
+    ARGS=$(echo "$REST" | sed -e 's/^/--build-arg /' -e 's/_/ --build-arg /g')
+  fi
+  TAG=$(echo "orc-$build" | sed -e 's/=/-/g')
+  ( cd $OS && docker build --no-cache -t "$TAG" $ARGS . )
 done
 echo "Start: $start"
 echo "End:" `date`

docker/run-all.sh

@@ -32,22 +32,24 @@ function failure {
 rm -f logs/pids.txt logs/*.log
 
 start=`date`
-for jdk in 8 11; do
-    for os in `cat os-list.txt`; do
-        if [[ "$os" = "debian10" && "$jdk" = "8" ]] || [[ "$os" = "debian9" && "$jdk" = "11" ]] || [[ "$os" = "ubuntu16" && "$jdk" = "11" ]]; then
-            echo "Skip building $os with $jdk"
-            continue
-        fi
-        echo "Building $os for $jdk"
-        ( cd $os && docker build -t "orc-$os-jdk${jdk}" --build-arg jdk=$jdk . ) > logs/${os}-jdk${jdk}-build.log 2>&1 || exit 1
-    done
+for build in `cat os-list.txt`; do
+  echo "Building $build"
+  OS=$(echo "$build" | cut -d '_' -f1)
+  REST=$(echo "$build" | cut -d '_' -f2- -s)
+  if [ -z "$REST" ]; then
+    ARGS=""
+  else
+    ARGS=$(echo "$REST" | sed -e 's/^/--build-arg /' -e 's/_/ --build-arg /g')
+  fi
+  TAG=$(echo "orc-$build" | sed -e 's/=/-/g')
+  ( cd $OS && docker build -t "$TAG" $ARGS . ) > logs/$build-build.log 2>&1 || exit 1
 done
 testStart=`date`
 
-for os in `cat os-list.txt`; do
-    ./run-one.sh $1 $2 $os > logs/$os-test.log 2>&1 &
+for build in `cat os-list.txt`; do
+    ./run-one.sh $1 $2 $build > logs/$build-test.log 2>&1 &
     echo "$!" >> logs/pids.txt
-    echo "Launching $os as $!"
+    echo "Launching $build as $!"
 done
 
 for job in `cat logs/pids.txt`; do

docker/run-one.sh

@@ -18,40 +18,36 @@
 GITHUB_USER=$1
 URL=https://github.com/$GITHUB_USER/orc.git
 BRANCH=$2
-OS=$3
+BUILD=$3
 
 function failure {
-    echo "FAILED $OS"
+    echo "FAILED $BUILD"
     exit 1
 }
 
 VOLUME="--volume m2cache:/root/.m2/repository"
+TAG=$(echo "orc-$BUILD" | sed -e 's/=/-/g')
 if [ $GITHUB_USER == "local" ]; then
   BRANCH=`git status| head -1 | sed -e 's/On branch //'`
-  echo "Started local run for $BRANCH on $OS at $(date)"
-  docker run $VOLUME -v`pwd`/..:/root/orc "orc-$OS" || failure
+  echo "Started local run for $BRANCH on $BUILD at $(date)"
+  docker run $VOLUME -v`pwd`/..:/root/orc "$TAG" || failure
 else
   CLONE="git clone $URL -b $BRANCH"
   MAKEDIR="mkdir orc/build && cd orc/build"
 
-  echo "Started $GITHUB_USER/$BRANCH on $OS at $(date)"
+  echo "Started $GITHUB_USER/$BRANCH on $BUILD at $(date)"
 
-  case $OS in
-  debian8)
+  case $BUILD in
+  debian8*)
      OPTS="-DSNAPPY_HOME=/usr/local"
      ;;
   *)
      OPTS=""
      ;;
   esac
 
-  for jdk in 8 11; do
-   if [[ "$OS" = "debian10" && "$jdk" = "8" ]] || [[ "$OS" = "debian9" && "$jdk" = "11" ]] || [[ "$OS" = "ubuntu16" && "$jdk" = "11" ]]; then
-      continue
-    fi
-   docker run $VOLUME "orc-$OS-jdk${jdk}" /bin/bash -c \
-      "$CLONE && $MAKEDIR && cmake $OPTS .. && make package test-out" \
-         || failure
-   done
+  docker run $VOLUME "$TAG" /bin/bash -c \
+     "$CLONE && $MAKEDIR && cmake $OPTS .. && make package test-out" \
+       || failure
 fi
-echo "Finished $OS at $(date)"
+echo "Finished $BUILD at $(date)"

docker/ubuntu20/Dockerfile

@@ -20,23 +20,36 @@
 FROM ubuntu:20.04
 LABEL maintainer="Apache ORC project <dev@orc.apache.org>"
 ARG jdk=8
+ARG cc=gcc
 
 RUN ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime
 RUN apt-get update
 RUN apt-get install -y \
   cmake \
-  gcc \
-  g++ \
   git \
   libsasl2-dev \
   libssl-dev \
   make \
   curl \
   maven \
   openjdk-${jdk}-jdk \
-  tzdata
+  tzdata; \
+  if [ "${cc}" = "gcc" ] ; then \
+    apt-get install -y \
+    gcc \
+    g++ \
+  ; else \
+    apt-get install -y \
+    clang \
+    && \
+    update-alternatives --set cc  /usr/bin/clang && \
+    update-alternatives --set c++ /usr/bin/clang++ \
+  ; fi
 RUN update-java-alternatives --set java-1.${jdk}.0-openjdk-amd64
 
+ENV CC=cc
+ENV CXX=c++
+
 WORKDIR /root
 VOLUME /root/.m2/repository