PIO-161 Spark 2.3.x support

.travis.yml

@@ -132,6 +132,35 @@ env:
       PIO_SPARK_VERSION=2.2.0
       PIO_ELASTICSEARCH_VERSION=5.5.2
 
+    - BUILD_TYPE=Unit
+      METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
+      PIO_SCALA_VERSION=2.11.12
+      PIO_SPARK_VERSION=2.3.1
+      PIO_HADOOP_VERSION=2.7.7
+    - BUILD_TYPE=Integration
+      METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
+      PIO_SCALA_VERSION=2.11.12
+      PIO_SPARK_VERSION=2.3.1
+      PIO_HADOOP_VERSION=2.7.7
+    - BUILD_TYPE=Integration
+      METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
+      PIO_SCALA_VERSION=2.11.12
+      PIO_SPARK_VERSION=2.3.1
+      PIO_ELASTICSEARCH_VERSION=1.7.3
+      PIO_HADOOP_VERSION=2.7.7
+    - BUILD_TYPE=Integration
+      METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
+      PIO_SCALA_VERSION=2.11.12
+      PIO_SPARK_VERSION=2.3.1
+      PIO_ELASTICSEARCH_VERSION=5.5.2
+      PIO_HADOOP_VERSION=2.7.7
+    - BUILD_TYPE=Integration
+      METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3
+      PIO_SCALA_VERSION=2.11.12
+      PIO_SPARK_VERSION=2.3.1
+      PIO_ELASTICSEARCH_VERSION=5.5.2
+      PIO_HADOOP_VERSION=2.7.7
+
     - BUILD_TYPE=LicenseCheck
 
 before_install:

build.sbt

@@ -44,6 +44,10 @@ lazy val scalaSparkDepsVersion = Map(
       "hadoop" -> "2.7.3",
       "json4s" -> "3.2.11"),
     "2.2" -> Map(
+      "akka" -> "2.4.17",
+      "hadoop" -> "2.7.3",
+      "json4s" -> "3.2.11"),
+    "2.3" -> Map(
       "akka" -> "2.4.17",
       "hadoop" -> "2.7.3",
       "json4s" -> "3.2.11")))

conf/pio-vendors.sh

@@ -51,7 +51,8 @@ PGSQL_DOWNLOAD=https://jdbc.postgresql.org/download/${PGSQL_JAR}
 HADOOP_MAJOR=`echo $PIO_HADOOP_VERSION | awk -F. '{print $1 "." $2}'`
 SPARK_DIR=spark-${PIO_SPARK_VERSION}-bin-hadoop${HADOOP_MAJOR}
 SPARK_ARCHIVE=${SPARK_DIR}.tgz
-SPARK_DOWNLOAD=http://d3kbcqa49mib13.cloudfront.net/${SPARK_ARCHIVE}
+SPARK_DOWNLOAD_MIRROR=https://www.apache.org/dyn/closer.lua\?action=download\&filename=spark/spark-${PIO_SPARK_VERSION}/${SPARK_ARCHIVE}
+SPARK_DOWNLOAD_ARCHIVE=https://archive.apache.org/dist/spark/spark-${PIO_SPARK_VERSION}/${SPARK_ARCHIVE}
 # ELASTICSEARCH_DOWNLOAD
 #   5.x https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${PIO_ELASTICSEARCH_VERSION}.tar.gz
 #   1.x https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${PIO_ELASTICSEARCH_VERSION}.tar.gz

tests/build_docker.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -ex
+#!/usr/bin/env bash
 
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -28,7 +28,10 @@ if [ ! -f $DIR/docker-files/${PGSQL_JAR} ]; then
   mv ${PGSQL_JAR} $DIR/docker-files/
 fi
 if [ ! -f $DIR/docker-files/${SPARK_ARCHIVE} ]; then
-  wget $SPARK_DOWNLOAD
+  curl -fLo $SPARK_ARCHIVE $SPARK_DOWNLOAD_MIRROR
+  if [[ $? -ne 0 ]]; then
+    curl -fLo $SPARK_ARCHIVE $SPARK_DOWNLOAD_ARCHIVE
+  fi
   mv $SPARK_ARCHIVE $DIR/docker-files/
 fi