Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion .github/workflows/velox_backend_x86.yml
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,15 @@ jobs:
matrix:
os: [ "ubuntu:20.04", "ubuntu:22.04" ]
spark: [ "spark-3.3", "spark-3.4", "spark-3.5", "spark-4.0", "spark-4.1" ]
java: [ "java-8", "java-11", "java-17", "java-21" ]
java: [ "java-8", "java-11", "java-17", "java-21", "java-25" ]
# Spark supports JDK17 since 3.3.
exclude:
- spark: spark-3.3
java: java-25
- spark: spark-3.4
java: java-25
- spark: spark-3.5
java: java-25
- spark: spark-3.3
java: java-21
- spark: spark-3.4
Expand All @@ -132,6 +138,8 @@ jobs:
java: java-11
- os: ubuntu:20.04
java: java-21
- os: ubuntu:20.04
java: java-25
- spark: spark-4.0
java: java-8
- spark: spark-4.0
Expand Down Expand Up @@ -167,6 +175,8 @@ jobs:
if [ "${{ matrix.java }}" = "java-17" ]; then
apt-get update && apt-get install -y openjdk-17-jdk wget
apt remove openjdk-11* -y
elif [ "${{ matrix.java }}" = "java-25" ]; then
apt-get update && apt-get install -y openjdk-25-jdk wget
elif [ "${{ matrix.java }}" = "java-21" ]; then
apt-get update && apt-get install -y openjdk-21-jdk wget
elif [ "${{ matrix.java }}" = "java-11" ]; then
Expand Down
5 changes: 3 additions & 2 deletions docs/developers/NewToGluten.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,13 @@ Gluten supports Ubuntu 20.04/22.04, CentOS 7/8, and MacOS.
Gluten supports JDK 8 for Spark 3.3, 3.4, and 3.5. For Spark 3.3 and later versions, Gluten
also supports JDK 11 and 17.

Note: Starting with Spark 4.0, the minimum required JDK version is 17.
Note: Starting with Spark 4.0, the minimum required JDK version is 17. JDK 21 and 25 are also
supported for Spark 4.0 and later versions.
We recommend using a higher JDK version now to ease migration when deploying Gluten for Spark 4.0
in the future. In addition, we may upgrade Arrow from 15.0.0 to a newer release, which will require
JDK 11 as the minimum version.

By default, Gluten compiles packages using JDK 8. Enable maven profile by `-Pjava-17` or `-Pjava-11` to use the corresponding JDK version, and ensure that the JDK version is available in your environment.
By default, Gluten compiles packages using JDK 8. Enable maven profile by `-Pjava-17`, `-Pjava-21`, `-Pjava-25` or `-Pjava-11` to use the corresponding JDK version, and ensure that the JDK version is available in your environment.

If JDK 11 or a higher version is used, Spark and Arrow require setting the java args `-Dio.netty.tryReflectionSetAccessible=true`, see [SPARK-29924](https://issues.apache.org/jira/browse/SPARK-29924) and [ARROW-6206](https://issues.apache.org/jira/browse/ARROW-6206).

Expand Down
21 changes: 19 additions & 2 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -1155,6 +1155,23 @@
<googlejavaformat.version>1.17.0</googlejavaformat.version>
</properties>
</profile>
<profile>
<id>java-25</id>
<activation>
<jdk>25</jdk>
</activation>
<properties>
<java.version>25</java.version>
<scala.release.version>21</scala.release.version>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<caffeine.version>3.1.8</caffeine.version>
<spotless.version>2.44.4</spotless.version>
<googlejavaformat.version>1.25.2</googlejavaformat.version>
<!-- spotless-maven-plugin 2.44.x is not compatible with JDK 25 due to internal API changes -->
<spotless.check.skip>true</spotless.check.skip>
</properties>
</profile>
<profile>
<id>spark-3.3</id>
<properties>
Expand Down Expand Up @@ -1298,7 +1315,7 @@
<rules>
<requireActiveProfile>
<!-- Spark 4.0 requires Java 17+ -->
<profiles>java-17,java-21</profiles>
<profiles>java-17,java-21,java-25</profiles>
<all>false</all>
<message>"-P spark-4.0" requires JDK 17+</message>
</requireActiveProfile>
Expand Down Expand Up @@ -1378,7 +1395,7 @@
<rules>
<requireActiveProfile>
<!-- Spark 4.1 requires Java 17+ -->
<profiles>java-17,java-21</profiles>
<profiles>java-17,java-21,java-25</profiles>
<all>false</all>
<message>"-P spark-4.1" requires JDK 17+</message>
</requireActiveProfile>
Expand Down
15 changes: 14 additions & 1 deletion tools/gluten-it/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,19 @@
<googlejavaformat.version>1.17.0</googlejavaformat.version>
</properties>
</profile>
<profile>
<id>java-25</id>
<activation>
<jdk>25</jdk>
</activation>
<properties>
<java.version>25</java.version>
<spotless.version>2.44.4</spotless.version>
<googlejavaformat.version>1.25.2</googlejavaformat.version>
<!-- spotless-maven-plugin 2.44.x is not compatible with JDK 25 due to internal API changes -->
<spotless.check.skip>true</spotless.check.skip>
</properties>
</profile>
<profile>
<id>spark-3.3</id>
<properties>
Expand Down Expand Up @@ -308,7 +321,7 @@
<id>spark-4.0</id>
<properties>
<spark.version>4.0.1</spark.version>
<scala.version>2.13.16</scala.version>
<scala.version>2.13.17</scala.version>
<scala.binary.version>2.13</scala.binary.version>
<delta.package.name>delta-spark</delta.package.name>
<delta.version>4.0.0</delta.version>
Expand Down
Loading