initial commit

.github/actions/setup-builder/action.yaml

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Prepare Builder
+description: 'Prepare Build Environment'
+inputs:
+  rust-version:
+    description: 'version of rust to install (e.g. nightly)'
+    required: true
+    default: 'nightly'
+  jdk-version:
+    description: 'jdk version to install (e.g., 17)'
+    required: true
+    default: '17'
+runs:
+  using: "composite"
+  steps:
+    - name: Install Build Dependencies
+      shell: bash
+      run: |
+        apt-get update
+        apt-get install -y openjdk-${{inputs.jdk-version}}-jdk protobuf-compiler
+    - name: Setup Rust toolchain
+      shell: bash
+      # rustfmt is needed for the substrait build script
+      run: |
+        echo "Installing ${{inputs.rust-version}}"
+        rustup toolchain install ${{ inputs.rust-version}}
+        rustup default ${{inputs.rust-version}}
+        rustup component add rustfmt clippy

.github/workflows/pr.yml

@@ -0,0 +1,114 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: PR Build
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+on:
+  push:
+    paths-ignore:
+      - "doc/**"
+      - "**.md"
+  pull_request:
+    paths-ignore:
+      - "doc/**"
+      - "**.md"
+  # manual trigger
+  # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
+  workflow_dispatch:
+
+env:
+  JAVA_VERSION: 17
+
+jobs:
+  linux-rust-test:
+    name: Rust tests (amd64)
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust & Java toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: nightly
+          jdk-version: ${{env.JAVA_VERSION}}
+
+      - name: Check cargo fmt
+        run: |
+          cd core
+          cargo fmt --all -- --check --color=never
+
+      - name: Check cargo clippy
+        run: |
+          cd core
+          cargo clippy --color=never -- -D warnings
+
+      - name: Check compilation
+        run: |
+          cd core
+          cargo check --benches
+
+      - name: Setup JAVA_HOME
+        run: |
+          echo "JAVA_HOME=/usr/lib/jvm/java-${{env.JAVA_VERSION}}-openjdk-amd64" >> $GITHUB_ENV
+
+      - name: Build common module (pre-requisite for Rust tests)
+        run: |
+          cd common
+          ../mvnw clean compile -DskipTests
+
+      - name: Run cargo test
+        run: |
+          # This is required to run some JNI related tests on the Rust side
+          echo "LIBRARY_PATH=$JAVA_HOME/lib:$JAVA_HOME/lib/server:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+          echo "$GITHUB_ENV"
+          cd core
+          cargo test --release
+
+  linux-java-test:
+    name: Java tests (amd64)
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust & Java toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: nightly
+          jdk-version: ${{env.JAVA_VERSION}}
+
+      - name: Run cargo build
+        run: |
+          cd core
+          cargo build
+
+      - name: Setup JAVA_HOME
+        run: |
+          echo "JAVA_HOME=/usr/lib/jvm/java-${{env.JAVA_VERSION}}-openjdk-amd64" >> $GITHUB_ENV
+
+      - name: Run Maven compile
+        run: |
+          ./mvnw compile test-compile scalafix:scalafix -Psemanticdb
+
+      - name: Run Maven install
+        run: |
+          SPARK_HOME=`pwd` ./mvnw clean install

pom.xml

@@ -64,6 +64,26 @@ under the License.
 
     <!-- Reverse default (skip installation), and then enable only for child modules -->
     <maven.deploy.skip>true</maven.deploy.skip>
+
+    <!-- For testing with JDK 17 -->
+    <extraJavaTestArgs>
+      -XX:+IgnoreUnrecognizedVMOptions
+      --add-opens=java.base/java.lang=ALL-UNNAMED
+      --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
+      --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
+      --add-opens=java.base/java.io=ALL-UNNAMED
+      --add-opens=java.base/java.net=ALL-UNNAMED
+      --add-opens=java.base/java.nio=ALL-UNNAMED
+      --add-opens=java.base/java.util=ALL-UNNAMED
+      --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
+      --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+      --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
+      --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+      --add-opens=java.base/sun.nio.cs=ALL-UNNAMED
+      --add-opens=java.base/sun.security.action=ALL-UNNAMED
+      --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
+      -Djdk.reflect.useDirectMethodHandle=false
+    </extraJavaTestArgs>
   </properties>
 
   <dependencyManagement>
@@ -623,6 +643,7 @@ under the License.
             <filereports>SparkTestSuite.txt</filereports>
             <stderr/>
             <tagsToExclude>org.apache.comet.IntegrationTestSuite</tagsToExclude>
+            <argLine>-ea -Xmx4g -Xss4m ${extraJavaTestArgs}</argLine>
             <systemProperties>
               <!-- emit test logs to target/unit-tests.log -->
               <log4j.configurationFile>file:src/test/resources/log4j2.properties</log4j.configurationFile>
@@ -664,6 +685,7 @@ under the License.
             <systemProperties>
               <log4j.configurationFile>file:src/test/resources/log4j2.properties</log4j.configurationFile>
             </systemProperties>
+            <argLine>-ea -Xmx4g -Xss4m ${extraJavaTestArgs}</argLine>
           </configuration>
         </plugin>
         <plugin>

spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala

@@ -297,7 +297,7 @@ class CometTPCDSV1_4_PlanStabilitySuite extends CometPlanStabilitySuite {
     new File(baseResourcePath, "approved-plans-v1_4").getAbsolutePath
 
   tpcdsQueries.foreach { q =>
-    test(s"check simplified (tpcds-v1.4/$q)") {
+    ignore(s"check simplified (tpcds-v1.4/$q)") {
       testQuery("tpcds", q)
     }
   }
@@ -308,7 +308,7 @@ class CometTPCDSV2_7_PlanStabilitySuite extends CometPlanStabilitySuite {
     new File(baseResourcePath, "approved-plans-v2_7").getAbsolutePath
 
   tpcdsQueriesV2_7_0.foreach { q =>
-    test(s"check simplified (tpcds-v2.7.0/$q)") {
+    ignore(s"check simplified (tpcds-v2.7.0/$q)") {
       testQuery("tpcds-v2.7.0", q)
     }
   }