diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 544bfc7a77d..3ae958f2b81 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -423,7 +423,7 @@ jobs: rm -rf spark/interpreter/metastore_db ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.5 -Pspark-scala-2.13 -Phadoop3 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} - livy-0-7-with-spark-3-4-1-under-python3: + livy-0-8-with-spark-3-4-1-under-python3: runs-on: ubuntu-20.04 steps: - name: Checkout @@ -450,7 +450,7 @@ jobs: run: | ./mvnw install -DskipTests -pl livy -am ${MAVEN_ARGS} ./testing/downloadSpark.sh "3.4.1" "3" - ./testing/downloadLivy.sh "0.7.1-incubating" + ./testing/downloadLivy.sh "0.8.0-incubating" "2.11" - name: Setup conda environment with python 3.9 and R uses: conda-incubator/setup-miniconda@v2 with: diff --git a/livy/README.md b/livy/README.md index 54311d93448..d3e76b62733 100644 --- a/livy/README.md +++ b/livy/README.md @@ -2,14 +2,14 @@ Livy interpreter for Apache Zeppelin # Prerequisities -You can follow the instructions at [Livy Quick Start](http://livy.io/quickstart.html) to set up livy. +You can follow the instructions at [Livy Get Started](https://livy.apache.org/get-started/) to set up livy. # Run Integration Tests -You can add integration test to [LivyInterpreter.java](https://github.com/apache/zeppelin/blob/master/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java) and run the integration test either via the CI environment or locally. You need to download livy-0.2 and spark-1.5.2 to local, then use the following script to run the integration test. +You can add integration test to [LivyInterpreter.java](https://github.com/apache/zeppelin/blob/master/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java) and run the integration test either via the CI environment or locally. You need to download livy-0.8 and spark-2.4.8 to local, then use the following script to run the integration test. ```bash #!/usr/bin/env bash -export LIVY_HOME= -export SPARK_HOME= +export LIVY_HOME= +export SPARK_HOME= ./mvnw clean verify -pl livy -DfailIfNoTests=false ``` diff --git a/livy/pom.xml b/livy/pom.xml index 70743f438b4..b600b50f621 100644 --- a/livy/pom.xml +++ b/livy/pom.xml @@ -39,7 +39,7 @@ 1.0.1.RELEASE - 0.7.1-incubating + 0.8.0-incubating 2.4.8 ${hadoop2.7.version} diff --git a/testing/downloadLivy.sh b/testing/downloadLivy.sh index 7f2faf3ffe0..f09837a7574 100755 --- a/testing/downloadLivy.sh +++ b/testing/downloadLivy.sh @@ -16,13 +16,21 @@ # limitations under the License. # -if [[ "$#" -ne 1 ]]; then - echo "usage) $0 [livy version]" - echo " eg) $0 0.2" +if [[ "$#" -ne 1 && "$#" -ne 2 ]]; then + echo "usage) $0 [scala version]" + echo " eg) $0 0.7.1-incubating" + echo " $0 0.8.0-incubating 2.11" exit 0 fi +# See simple version normalization: +# http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers +function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; } LIVY_VERSION="${1}" +SCALA_VERSION_SUFFIX="" +if [ $(version $LIVY_VERSION) -ge $(version "0.8.0") ]; then + SCALA_VERSION_SUFFIX="_${2}" +fi set -xe @@ -49,7 +57,7 @@ download_with_retry() { } LIVY_CACHE=".livy-dist" -LIVY_ARCHIVE="apache-livy-${LIVY_VERSION}-bin" +LIVY_ARCHIVE="apache-livy-${LIVY_VERSION}${SCALA_VERSION_SUFFIX}-bin" export LIVY_HOME="${ZEPPELIN_HOME}/livy-server-$LIVY_VERSION" echo "LIVY_HOME is ${LIVY_HOME}"