Uniffle ga

.github/workflows/build-arm-releases.yml

@@ -50,6 +50,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - uses: actions/setup-java@v4
         with:
           distribution: 'adopt-hotspot'
@@ -61,7 +63,7 @@ jobs:
           version: "21.7"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - uses:  ./.github/actions/setup-rust-toolchain
         with:
           rustflags: --allow warnings -C target-feature=+aes
           components:

.github/workflows/build-ce7-releases.yml

@@ -48,8 +48,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - uses: s4u/setup-maven-action@main
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - uses:  ./.github/actions/setup-rust-toolchain
 
       - uses: prompt/actions-commit-hash@v3
         id: commit

.github/workflows/tpcds-reusable.yml

@@ -42,6 +42,22 @@ on:
         required: false
         type: string
         default: ''
+      unifflever:
+        required: false
+        type: string
+        default: ''
+      uniffleurl:
+        required: false
+        type: string
+        default: ''
+      hadoopver:
+        required: false
+        type: string
+        default: ''
+      hadoopurl:
+        required: false
+        type: string
+        default: ''
       extrabuildopt:
         required: false
         type: string
@@ -110,6 +126,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - uses: actions/setup-java@v4
         with:
           distribution: 'adopt-hotspot'
@@ -121,16 +139,13 @@ jobs:
           version: "21.7"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - uses:  ./.github/actions/setup-rust-toolchain
         with:
           rustflags: --allow warnings -C target-feature=+aes
           components:
             cargo
             rustfmt
 
-      - name: Rustfmt Check
-        uses: actions-rust-lang/rustfmt@v1
-
       - name: Cargo test
         run: |
           cargo --version
@@ -217,10 +232,10 @@ jobs:
       - name: Start Celeborn-${{ inputs.celebornver }}
         if: ${{ inputs.celebornver != '' && inputs.celebornurl != '' }}
         run: |
-          mkdir -p /tmp/celeborn/data && mkdir -p /tmp/celeborn/logs && \
+          mkdir -p /tmp/rss/data && mkdir -p /tmp/rss/logs && \
           cd celeborn-bin-${{ inputs.celebornver }} && \
-          bash -c "echo -e 'CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g\nCELEBORN_LOG_DIR=/tmp/celeborn/logs' > ./conf/celeborn-env.sh" && \
-          bash -c "echo -e 'celeborn.worker.storage.dirs /tmp/celeborn\nceleborn.worker.storage.workingDir data\nceleborn.worker.commitFiles.threads 128\nceleborn.worker.sortPartition.threads 64' > ./conf/celeborn-defaults.conf" && \
+          bash -c "echo -e 'CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g\nCELEBORN_LOG_DIR=/tmp/rss/logs' > ./conf/celeborn-env.sh" && \
+          bash -c "echo -e 'celeborn.worker.storage.dirs /tmp/rss\nceleborn.worker.storage.workingDir data\nceleborn.worker.commitFiles.threads 128\nceleborn.worker.sortPartition.threads 64' > ./conf/celeborn-defaults.conf" && \
           bash ./sbin/start-master.sh && \
           bash ./sbin/start-worker.sh
 
@@ -230,6 +245,53 @@ jobs:
           ls -la celeborn-bin-${{ inputs.celebornver }}/spark
           cp celeborn-bin-${{ inputs.celebornver }}/spark/celeborn-client-spark-*_${{ inputs.scalaver }}-*.jar spark-bin-${{ inputs.sparkver }}_${{ inputs.scalaver }}/jars/
 
+      - uses: actions/cache@v4
+        if: ${{ inputs.unifflever != '' && inputs.uniffleurl != '' }}
+        id: cache-uniffle-bin
+        with:
+          path: uniffle-bin-${{ inputs.unifflever }}
+          key: uniffle-bin-${{ inputs.unifflever }}
+
+      - name: Setup Uniffle-${{ inputs.unifflever }}
+        id: setup-uniffle-bin
+        if: ${{ inputs.unifflever != '' && inputs.uniffleurl != '' &&  steps.cache-uniffle-bin.outputs.cache-hit != 'true' }}
+        run: |
+          wget -c ${{ inputs.uniffleurl }} && \
+          mkdir -p uniffle-bin-${{ inputs.unifflever }} && \
+          tar -xf ./apache-uniffle-${{ inputs.unifflever }}-incubating-bin.tar.gz -C uniffle-bin-${{ inputs.unifflever }} --strip-component=1
+
+      - uses: actions/cache@v4
+        if: ${{ inputs.hadoopver != '' && inputs.hadoopurl != '' }}
+        id: cache-hadoop-bin
+        with:
+          path: hadoop-bin-${{ inputs.hadoopver }}
+          key: hadoop-bin-${{ inputs.hadoopver }}
+
+      - name: Setup hadoop-${{ inputs.hadoopver }}
+        id: setup-hadoop-bin
+        if: ${{ inputs.hadoopver != '' && inputs.hadoopurl != '' &&  steps.cache-hadoop-bin.outputs.cache-hit != 'true' }}
+        run: |
+          wget -c ${{ inputs.hadoopurl }} && \
+          mkdir -p hadoop-bin-${{ inputs.hadoopver }} && \
+          tar -xf ./hadoop-${{ inputs.hadoopver }}.tar.gz -C hadoop-bin-${{ inputs.hadoopver }} --strip-component=1  && \
+          ls -la hadoop-bin-${{ inputs.hadoopver }}
+
+      - name: Start Uniffle-${{ inputs.unifflever }}
+        if: ${{ inputs.unifflever != '' && inputs.uniffleurl != '' }}
+        run: |
+          mkdir -p /tmp/rss/data && mkdir -p /tmp/rss/logs && \
+          cd uniffle-bin-${{ inputs.unifflever }} && \
+          bash -c "echo -e 'XMX_SIZE=16g\nHADOOP_HOME=../hadoop-bin-${{ inputs.hadoopver }}\nRSS_LOG_DIR=/tmp/rss/logs' > ./bin/rss-env.sh" && \
+          bash -c "echo -e 'rss.coordinator.shuffle.nodes.max 1\nrss.rpc.server.port 19999' > ./conf/coordinator.conf" && \
+          bash -c "echo -e 'rss.server.app.expired.withoutHeartbeat 7200000\nrss.server.heartbeat.delay 3000\nrss.rpc.server.port 19997\nrss.rpc.server.type GRPC_NETTY\nrss.jetty.http.port 19996\nrss.server.netty.port 19995\nrss.storage.basePath /tmp/rss/data\nrss.storage.type MEMORY_LOCALFILE\nrss.coordinator.quorum localhost:19999\nrss.server.flush.thread.alive 10\nrss.server.single.buffer.flush.threshold 64m' > ./conf/server.conf" && \
+          bash ./bin/start-coordinator.sh && bash ./bin/start-shuffle-server.sh
+
+      - name: Install Uniffle JAR
+        if: ${{ inputs.unifflever != '' && inputs.uniffleurl != '' }}
+        run: |
+          ls -la uniffle-bin-${{ inputs.unifflever }}/jars/client/spark3/
+          cp uniffle-bin-${{ inputs.unifflever }}/jars/client/spark3/*.jar spark-bin-${{ inputs.sparkver }}_${{ inputs.scalaver }}/jars/
+
       - name: Run
         run: |
           ls -la
@@ -240,3 +302,11 @@ jobs:
             ${{ inputs.extrasparkconf }} \
             --data-location dev/tpcds_1g \
             --query-filter ${{ matrix.query }}
+
+      - name: Upload RSS log
+        if: ${{ failure() && (inputs.celebornver != '' || inputs.unifflever != '') }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: rss-log-${{ inputs.extraidentifier }}
+          path: |
+            /tmp/rss/logs/*

.github/workflows/uniffle.yml

@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Uniffle
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - master
+      - branch-*
+
+concurrency:
+  group: uniffle-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  test-uniffle:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - unifflever: "0.9.2"
+            hadoopver: "2.8.5"
+            uniffleprofile: "uniffle,uniffle-0.9"
+    uses: ./.github/workflows/tpcds-reusable.yml
+    name: Test Uniffle ${{ matrix.unifflever }}
+    with:
+      unifflever: ${{ matrix.unifflever }}
+      uniffleurl: https://archive.apache.org/dist/uniffle/${{ matrix.unifflever }}/apache-uniffle-${{ matrix.unifflever }}-incubating-bin.tar.gz
+      hadoopver: ${{ matrix.hadoopver }}
+      hadoopurl: https://archive.apache.org/dist/hadoop/common/hadoop-${{ matrix.hadoopver }}/hadoop-${{ matrix.hadoopver }}.tar.gz
+      extrabuildopt: -P${{ matrix.uniffleprofile }} -DuniffleVersion=${{ matrix.unifflever }}
+      extraidentifier: uniffle-${{ matrix.unifflever }}
+      sparkver: "spark-3.5"
+      sparkurl: "https://archive.apache.org/dist/spark/spark-3.5.6/spark-3.5.6-bin-hadoop3.tgz"
+      scalaver: "2.12"
+      extrasparkconf: >-
+        --conf spark.shuffle.manager=org.apache.spark.sql.execution.auron.shuffle.uniffle.AuronUniffleShuffleManager
+        --conf spark.serializer=org.apache.spark.serializer.KryoSerializer
+        --conf spark.rss.coordinator.quorum=localhost:19999
+        --conf spark.rss.enabled=true
+        --conf spark.rss.storage.type=MEMORY_LOCALFILE
+        --conf spark.rss.client.type=GRPC_NETTY
+      queries: '["q1,q2,q3,q4,q5,q6,q7,q8,q9"]'

.gitmodules

@@ -17,4 +17,7 @@
 
 [submodule "dev/tpcds_1g"]
 	path = dev/tpcds_1g
-	url = git@github.com:blaze-init/tpcds_1g.git
+	url = https://github.com/auron-project/tpcds_1g
+[submodule ".github/actions/setup-rust-toolchain"]
+	path = .github/actions/setup-rust-toolchain
+	url = https://github.com/actions-rust-lang/setup-rust-toolchain