adding volume for spark3.3.1 build

Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
apache · Jun 6, 2023 · 3d713dd · 3d713dd
1 parent 381a91c
commit 3d713dd
Showing 1 changed file with 9 additions and 2 deletions.
diff --git a/.github/workflows/velox_be.yml b/.github/workflows/velox_be.yml
@@ -168,14 +168,21 @@ jobs:
       - uses: actions/checkout@v2
       - name: Setup docker container
         run: |
-          EXTRA_DOCKER_OPTIONS="--name velox-backend-ubuntu2004-test-spark33-$GITHUB_RUN_ID -e NUM_THREADS=30 --detach" \
+          EXTRA_DOCKER_OPTIONS="--name velox-backend-ubuntu2004-test-spark33-$GITHUB_RUN_ID --volume velox-backend-spark33-cache:/var/cache/spark33 -e NUM_THREADS=30 --detach" \
           NON_INTERACTIVE=ON \
           MOUNT_MAVEN_CACHE=OFF \
           OS_IMAGE=ubuntu:20.04 \
           tools/gluten-te/ubuntu/cbash.sh sleep 14400
       - name: Setup maven cache
         run: |
           docker cp ~/.m2/repository velox-backend-ubuntu2004-test-spark33-$GITHUB_RUN_ID:/root/.m2/
+      - name: Compile Spark3.3
+        run: |
+          docker exec velox-backend-ubuntu2004-test-spark33-$GITHUB_RUN_ID bash -c '
+          cd /var/cache/spark33 && \
+          git clone --depth 1 --branch v3.3.1 https://github.com/apache/spark.git spark331 \
+          cd /var/cache/spark33 && ./build/mvn -Pyarn -DskipTests install
+          '
       - name: Build Gluten Velox-backend third party
         run: |
           docker exec velox-backend-ubuntu2004-test-spark33-$GITHUB_RUN_ID bash -c '
@@ -198,7 +205,7 @@ jobs:
       - name: Build and Run unit test for Spark 3.3.1
         run: |
           docker exec velox-backend-ubuntu2004-test-spark33-$GITHUB_RUN_ID bash -c 'cd /opt/gluten && \
-          mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Pspark-ut'
+          mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Pspark-ut  -DargLine="-Dspark.test.home=/var/cache/spark33"'
       - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.3
         run: |
           docker exec velox-backend-ubuntu2004-test-spark33-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/tools/gluten-it && \