[ZEPPELIN-5973] Bump Livy 0.8.0-incubating

.github/workflows/core.yml

@@ -423,7 +423,7 @@ jobs:
           rm -rf spark/interpreter/metastore_db
           ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.5 -Pspark-scala-2.13 -Phadoop3 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
 
-  livy-0-7-with-spark-3-4-1-under-python3:
+  livy-0-8-with-spark-3-4-1-under-python3:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
@@ -450,7 +450,7 @@ jobs:
         run: |
           ./mvnw install -DskipTests -pl livy -am  ${MAVEN_ARGS}
           ./testing/downloadSpark.sh "3.4.1" "3"
-          ./testing/downloadLivy.sh "0.7.1-incubating"
+          ./testing/downloadLivy.sh "0.8.0-incubating" "2.11"
       - name: Setup conda environment with python 3.9 and R
         uses: conda-incubator/setup-miniconda@v2
         with:

livy/README.md

@@ -2,14 +2,14 @@
 Livy interpreter for Apache Zeppelin
 
 # Prerequisities
-You can follow the instructions at [Livy Quick Start](http://livy.io/quickstart.html) to set up livy.
+You can follow the instructions at [Livy Get Started](https://livy.apache.org/get-started/) to set up livy.
 
 # Run Integration Tests
-You can add integration test to [LivyInterpreter.java](https://github.com/apache/zeppelin/blob/master/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java) and run the integration test either via the CI environment or locally. You need to download livy-0.2 and spark-1.5.2 to local, then use the following script to run the integration test.
+You can add integration test to [LivyInterpreter.java](https://github.com/apache/zeppelin/blob/master/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java) and run the integration test either via the CI environment or locally. You need to download livy-0.8 and spark-2.4.8 to local, then use the following script to run the integration test.
 
 ```bash
 #!/usr/bin/env bash
-export LIVY_HOME=<path_of_livy_0.2.0>
-export SPARK_HOME=<path_of_spark-1.5.2>
+export LIVY_HOME=<path_of_livy_0.8.0>
+export SPARK_HOME=<path_of_spark-2.4.8>
 ./mvnw clean verify -pl livy -DfailIfNoTests=false
 ```

livy/pom.xml

@@ -39,7 +39,7 @@
         <spring.security.kerberosclient>1.0.1.RELEASE</spring.security.kerberosclient>
 
         <!--test library versions-->
-        <livy.version>0.7.1-incubating</livy.version>
+        <livy.version>0.8.0-incubating</livy.version>
         <spark.version>2.4.8</spark.version>
         <hadoop.version>${hadoop2.7.version}</hadoop.version>
     </properties>

testing/downloadLivy.sh

@@ -16,13 +16,21 @@
 # limitations under the License.
 #
 
-if [[ "$#" -ne 1 ]]; then
-    echo "usage) $0 [livy version]"
-    echo "   eg) $0 0.2"
+if [[ "$#" -ne 1 && "$#" -ne 2 ]]; then
+    echo "usage) $0 <livy version> [scala version]"
+    echo "   eg) $0 0.7.1-incubating"
+    echo "       $0 0.8.0-incubating 2.11"
     exit 0
 fi
 
+# See simple version normalization:
+# http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers
+function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; }
 LIVY_VERSION="${1}"
+SCALA_VERSION_SUFFIX=""
+if [ $(version $LIVY_VERSION) -ge $(version "0.8.0") ]; then
+    SCALA_VERSION_SUFFIX="_${2}"
+fi
 
 set -xe
 
@@ -49,7 +57,7 @@ download_with_retry() {
 }
 
 LIVY_CACHE=".livy-dist"
-LIVY_ARCHIVE="apache-livy-${LIVY_VERSION}-bin"
+LIVY_ARCHIVE="apache-livy-${LIVY_VERSION}${SCALA_VERSION_SUFFIX}-bin"
 export LIVY_HOME="${ZEPPELIN_HOME}/livy-server-$LIVY_VERSION"
 echo "LIVY_HOME is ${LIVY_HOME}"