From d8bcba889cde3cfd7c48e321ed72768c5f17544f Mon Sep 17 00:00:00 2001 From: peng Date: Thu, 19 Oct 2023 11:14:22 -0400 Subject: [PATCH] upgrade CI scripts, local-cluster mode is enabled again --- .circleci/config.yml | 14 +++++++------- .gitattributes | 1 + .scalafix.conf | 1 - dev/CI/main.sh | 2 +- dev/profiles/apache-latest/.common.sh | 5 +++-- dev/profiles/apache-local/.common.sh | 7 +++---- dev/profiles/apache-stable/.common.sh | 5 +++-- dev/{CI => }/update-submodules.sh | 0 8 files changed, 18 insertions(+), 17 deletions(-) rename dev/{CI => }/update-submodules.sh (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4ad21cb47..7de6143f0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,14 +1,14 @@ version: 2.1 executors: - openjdk11: + main: docker: - image: cimg/openjdk:11.0 resource_class: large jobs: build: - executor: openjdk11 + executor: main steps: - checkout - run: @@ -17,22 +17,22 @@ jobs: echo "Starting ..." - restore_cache: keys: - - profile-{{ checksum "./dev/profiles/apache-local/.common.sh" }} + - profile-{{ checksum "./dev/profiles/apache-latest/.common.sh" }} - run: name: Prepare command: | - ./dev/CI-apache-local.sh prepare + ./dev/CI-apache-latest.sh prepare - save_cache: - key: profile-{{ checksum "./dev/profiles/apache-local/.common.sh" }} + key: profile-{{ checksum "./dev/profiles/apache-latest/.common.sh" }} paths: - ~/.ci - run: name: Run command: | - ./dev/CI-apache-local.sh + ./dev/CI-apache-latest.sh workflows: - main-local: # This is the name of the workflow, feel free to change it to better match your workflow. + main: # Inside the workflow, you define the jobs you want to run. # For more details on extending your workflow, see the configuration docs: https://circleci.com/docs/2.0/configuration-reference/#workflows jobs: diff --git a/.gitattributes b/.gitattributes index ccd5ce61e..856227182 100644 --- a/.gitattributes +++ b/.gitattributes @@ -7,3 +7,4 @@ *.R text eol=lf # from apache spark +*.sh text eol=lf diff --git a/.scalafix.conf b/.scalafix.conf index 8b9db2eec..5d3dd64e7 100644 --- a/.scalafix.conf +++ b/.scalafix.conf @@ -15,7 +15,6 @@ RemoveUnused { imports = true privates = true locals = true - // the following 2 are not compatible with scala 2.11 patternvars = true params = true } diff --git a/dev/CI/main.sh b/dev/CI/main.sh index c84e80990..8dbbb2a5b 100755 --- a/dev/CI/main.sh +++ b/dev/CI/main.sh @@ -9,5 +9,5 @@ if [ "${2}" = "prepare" ]; then exit 0 fi -"$FWDIR"/CI/update-submodules.sh && \ +"$FWDIR"/update-submodules.sh && \ "$FWDIR"/CI/pipeline.sh "-PnotLocal" "${BUILD_PROFILES[@]}" diff --git a/dev/profiles/apache-latest/.common.sh b/dev/profiles/apache-latest/.common.sh index 2e424020c..a24632da8 100644 --- a/dev/profiles/apache-latest/.common.sh +++ b/dev/profiles/apache-latest/.common.sh @@ -3,7 +3,8 @@ SPARK_NAME="${SPARK:-spark-3.5.0}" SPARK_DIR_ROOT="$HOME/.ci/spark-dist" -SPARK_DIR_NAME="$SPARK_NAME"-bin-hadoop3-scala2.13 +export SPARK_SCALA_VERSION="2.13" +SPARK_DIR_NAME="$SPARK_NAME"-bin-hadoop3-scala"${SPARK_SCALA_VERSION}" SPARK_URL="http://archive.apache.org/dist/spark/${SPARK_NAME}/${SPARK_DIR_NAME}.tgz" @@ -13,4 +14,4 @@ tar -xzf "$SPARK_DIR_ROOT/$SPARK_DIR_NAME".tgz -C "$SPARK_DIR_ROOT" export SPARK_HOME="$SPARK_DIR_ROOT/$SPARK_DIR_NAME" -export BUILD_PROFILES=("-PsparkVersion=3.5.0" "-PscalaVersion=2.13.12") +export BUILD_PROFILES=("-PsparkVersion=3.5.0") diff --git a/dev/profiles/apache-local/.common.sh b/dev/profiles/apache-local/.common.sh index 4e6527b85..02ef043af 100644 --- a/dev/profiles/apache-local/.common.sh +++ b/dev/profiles/apache-local/.common.sh @@ -3,7 +3,8 @@ SPARK_NAME="${SPARK:-spark-3.5.0}" SPARK_DIR_ROOT="$HOME/.ci/spark-dist" -SPARK_DIR_NAME="$SPARK_NAME"-bin-hadoop3-scala2.13 +export SPARK_SCALA_VERSION="2.13" +SPARK_DIR_NAME="$SPARK_NAME"-bin-hadoop3-scala"${SPARK_SCALA_VERSION}" SPARK_URL="http://archive.apache.org/dist/spark/${SPARK_NAME}/${SPARK_DIR_NAME}.tgz" @@ -11,6 +12,4 @@ SPARK_URL="http://archive.apache.org/dist/spark/${SPARK_NAME}/${SPARK_DIR_NAME}. wget -N "$SPARK_URL" -P "$SPARK_DIR_ROOT" tar -xzf "$SPARK_DIR_ROOT/$SPARK_DIR_NAME".tgz -C "$SPARK_DIR_ROOT" -#export SPARK_HOME="$SPARK_DIR_ROOT/$SPARK_DIR_NAME" - -export BUILD_PROFILES=("-PsparkVersion=3.5.0" "-PscalaVersion=2.13.12") +export BUILD_PROFILES=("-PsparkVersion=3.5.0") diff --git a/dev/profiles/apache-stable/.common.sh b/dev/profiles/apache-stable/.common.sh index 2e424020c..a24632da8 100644 --- a/dev/profiles/apache-stable/.common.sh +++ b/dev/profiles/apache-stable/.common.sh @@ -3,7 +3,8 @@ SPARK_NAME="${SPARK:-spark-3.5.0}" SPARK_DIR_ROOT="$HOME/.ci/spark-dist" -SPARK_DIR_NAME="$SPARK_NAME"-bin-hadoop3-scala2.13 +export SPARK_SCALA_VERSION="2.13" +SPARK_DIR_NAME="$SPARK_NAME"-bin-hadoop3-scala"${SPARK_SCALA_VERSION}" SPARK_URL="http://archive.apache.org/dist/spark/${SPARK_NAME}/${SPARK_DIR_NAME}.tgz" @@ -13,4 +14,4 @@ tar -xzf "$SPARK_DIR_ROOT/$SPARK_DIR_NAME".tgz -C "$SPARK_DIR_ROOT" export SPARK_HOME="$SPARK_DIR_ROOT/$SPARK_DIR_NAME" -export BUILD_PROFILES=("-PsparkVersion=3.5.0" "-PscalaVersion=2.13.12") +export BUILD_PROFILES=("-PsparkVersion=3.5.0") diff --git a/dev/CI/update-submodules.sh b/dev/update-submodules.sh similarity index 100% rename from dev/CI/update-submodules.sh rename to dev/update-submodules.sh