[VL] move Spark3.3 Unit tests to seperate job

This patch moved Spark 3.3 unit tests to a seperate job to speed up GHA Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
apache · Jun 5, 2023 · bb428ec · bb428ec
1 parent d84ca65
commit bb428ec
Showing 1 changed file with 41 additions and 9 deletions.
diff --git a/.github/workflows/velox_be.yml b/.github/workflows/velox_be.yml
@@ -157,29 +157,61 @@ jobs:
             --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=20g -s=1.0 --cpus=16 --iterations=1 \
           && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh \
             --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=20g -s=1.0 --cpus=16 --iterations=1'
+      - name: Exit docker container
+        if: ${{ always() }}
+        run: |
+          docker stop velox-backend-ubuntu2004-test-slow-$GITHUB_RUN_ID || true
+
+  velox-backend-ubuntu2004-test-spark33:
+    runs-on: velox-self-hosted
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup docker container
+        run: |
+          EXTRA_DOCKER_OPTIONS="--name velox-backend-ubuntu2004-test-spark33-$GITHUB_RUN_ID -e NUM_THREADS=30 --detach" \
+          NON_INTERACTIVE=ON \
+          MOUNT_MAVEN_CACHE=OFF \
+          OS_IMAGE=ubuntu:20.04 \
+          tools/gluten-te/ubuntu/cbash.sh sleep 14400
+      - name: Setup maven cache
+        run: |
+          docker cp ~/.m2/repository velox-backend-ubuntu2004-test-spark33-$GITHUB_RUN_ID:/root/.m2/
+      - name: Build Gluten Velox-backend third party
+        run: |
+          docker exec velox-backend-ubuntu2004-test-spark33-$GITHUB_RUN_ID bash -c '
+          cd /opt/gluten/ep/build-arrow/src && \
+          ./get_arrow.sh  && \
+          ./build_arrow.sh --build_tests=ON --build_benchmarks=ON && \
+          cd /opt/gluten/ep/build-velox/src && \
+          ./get_velox.sh && \
+          ./build_velox.sh'
+      - name: Build Gluten CPP library
+        run: |
+          docker exec velox-backend-ubuntu2004-test-spark33-$GITHUB_RUN_ID bash -c '
+          cd /opt/gluten/cpp && \
+          mkdir build && cd build && cmake .. && make -j'
       - name: Build for Spark 3.3.0
         run: |
-          docker exec velox-backend-ubuntu2004-test-slow-$GITHUB_RUN_ID bash -c '
+          docker exec velox-backend-ubuntu2004-test-spark33-$GITHUB_RUN_ID bash -c '
           cd /opt/gluten && \
           mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -DskipTests -Dspark33.version=3.3.0'
       - name: Build and Run unit test for Spark 3.3.1
         run: |
-          docker exec velox-backend-ubuntu2004-test-slow-$GITHUB_RUN_ID bash -c 'cd /opt/gluten && \
+          docker exec velox-backend-ubuntu2004-test-spark33-$GITHUB_RUN_ID bash -c 'cd /opt/gluten && \
           mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Pspark-ut'
       - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.3
         run: |
-          docker exec velox-backend-ubuntu2004-test-slow-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/tools/gluten-it && \
+          docker exec velox-backend-ubuntu2004-test-spark33-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/tools/gluten-it && \
           mvn clean package -Pspark-3.3 \
           && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh \
-            --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=20g -s=1.0 --cpus=16 --iterations=1 --skip-data-gen \
+            --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=20g -s=1.0 --cpus=16 --iterations=1 \
           && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh \
-            --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=20g -s=1.0 --cpus=16 --iterations=1 --skip-data-gen'
+            --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=20g -s=1.0 --cpus=16 --iterations=1'
       - name: Exit docker container
         if: ${{ always() }}
         run: |
-          docker stop velox-backend-ubuntu2004-test-slow-$GITHUB_RUN_ID || true
-
-
+          docker stop velox-backend-ubuntu2004-test-spark33-$GITHUB_RUN_ID || true
+         
   velox-backend-ubuntu2204-test:
     runs-on: velox-self-hosted
     steps:
@@ -421,4 +453,4 @@ jobs:
       - name: Exit docker container
         if: ${{ always() }}
         run: |
-          docker stop velox-backend-static-build-test-$GITHUB_RUN_ID || true
+          docker stop velox-backend-static-build-test-$GITHUB_RUN_ID || true