Skip to content

Spark: Add support for 4.2.0 #14984

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
manuzhang wants to merge 3 commits into
base: main
Choose a base branch
from
+153,477 −14
Open

Spark: Add support for 4.2.0 #14984

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ed56e7c
Spark: Move 4.1 as 4.2
manuzhang Aug 3, 2026
426764d
Spark: Copy back 4.2 as 4.1
manuzhang Aug 3, 2026
cccba25
Spark: Initial support for 4.2.0
manuzhang Aug 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
6 changes: 6 additions & 0 deletions .github/workflows/cve-scan.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,12 @@ jobs:
:iceberg-spark:iceberg-spark-runtime-4.1_2.13:shadowJar
scan-path: spark/v4.1/spark-runtime/build/libs
unpack: false
- distribution: spark-runtime-4.2_2.13
build-task: >-
-DsparkVersions=4.2
:iceberg-spark:iceberg-spark-runtime-4.2_2.13:shadowJar
scan-path: spark/v4.2/spark-runtime/build/libs
unpack: false
- distribution: flink-runtime-1.20
build-task: >-
-DflinkVersions=1.20
Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/spark-ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ jobs:
matrix:
jvm: [17, 21]
event_name: ['${{ github.event_name }}']
spark: ['3.5', '4.0', '4.1']
spark: ['3.5', '4.0', '4.1', '4.2']
scala: ['2.12', '2.13']
# Split iceberg-spark (core) from iceberg-spark-extensions/-runtime
# so they run as concurrent jobs rather than serially in one job.
Expand All @@ -95,6 +95,8 @@ jobs:
scala: '2.12'
- spark: '4.1'
scala: '2.12'
- spark: '4.2'
scala: '2.12'
# Pull requests run the baseline JDK only.
- event_name: pull_request
jvm: 21
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ spark/v4.0/spark/benchmark/*
spark/v4.0/spark-extensions/benchmark/*
spark/v4.1/spark/benchmark/*
spark/v4.1/spark-extensions/benchmark/*
spark/v4.2/spark/benchmark/*
spark/v4.2/spark-extensions/benchmark/*
*/benchmark/*

__pycache__/
Expand Down
4 changes: 2 additions & 2 deletions dev/stage-binaries.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,13 @@

SCALA_VERSION=2.12
FLINK_VERSIONS=1.20,2.0,2.1
SPARK_VERSIONS=3.5,4.0,4.1
SPARK_VERSIONS=3.5,4.0,4.1,4.2
KAFKA_VERSIONS=3

./gradlew -Prelease -DscalaVersion=$SCALA_VERSION -DflinkVersions=$FLINK_VERSIONS -DsparkVersions=$SPARK_VERSIONS -DkafkaVersions=$KAFKA_VERSIONS publishApachePublicationToMavenRepository --no-parallel --no-configuration-cache

# Also publish Scala 2.13 Artifacts for versions that support it.
# Flink does not yet support 2.13 (and is largely dropping a user-facing dependency on Scala). Hive doesn't need a Scala specification.
./gradlew -Prelease -DscalaVersion=2.13 -DsparkVersions=3.5 :iceberg-spark:iceberg-spark-3.5_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-extensions-3.5_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-runtime-3.5_2.13:publishApachePublicationToMavenRepository --no-parallel --no-configuration-cache
# Spark 4.0 only supports Scala 2.13. no need to specify scalaVersion
# Spark 4.0+ only supports Scala 2.13. no need to specify scalaVersion

Comment thread
manuzhang marked this conversation as resolved.
4 changes: 2 additions & 2 deletions gradle.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ jmhJsonOutputPath=build/reports/jmh/results.json
jmhIncludeRegex=.*
systemProp.defaultFlinkVersions=2.1
systemProp.knownFlinkVersions=1.20,2.0,2.1
systemProp.defaultSparkVersions=4.1
systemProp.knownSparkVersions=3.5,4.0,4.1
systemProp.defaultSparkVersions=4.2
systemProp.knownSparkVersions=3.5,4.0,4.1,4.2
systemProp.defaultKafkaVersions=3
systemProp.knownKafkaVersions=3
systemProp.defaultScalaVersion=2.12
Expand Down
1 change: 1 addition & 0 deletions gradle/libs.versions.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ snowflake-jdbc = "3.28.0"
spark35 = "3.5.9"
spark40 = "4.0.4"
spark41 = "4.1.3"
spark42 = "4.2.0"
sqlite-jdbc = "3.53.2.1"
testcontainers = "2.0.5"
tez08 = { strictly = "0.8.4"} # see rich version usage explanation above
Expand Down
5 changes: 5 additions & 0 deletions jmh.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@ if (sparkVersions.contains("4.1")) {
jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-4.1_2.13"))
}

if (sparkVersions.contains("4.2")) {
jmhProjects.add(project(":iceberg-spark:iceberg-spark-4.2_2.13"))
jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-4.2_2.13"))
}

configure(jmhProjects) {
apply plugin: 'me.champeau.jmh'
apply plugin: 'io.morethan.jmhreport'
Expand Down
30 changes: 21 additions & 9 deletions settings.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,15 +164,27 @@ if (sparkVersions.contains("4.0")) {
}

if (sparkVersions.contains("4.1")) {
include ":iceberg-spark:spark-4.1_2.13"
include ":iceberg-spark:spark-extensions-4.1_2.13"
include ":iceberg-spark:spark-runtime-4.1_2.13"
project(":iceberg-spark:spark-4.1_2.13").projectDir = file('spark/v4.1/spark')
project(":iceberg-spark:spark-4.1_2.13").name = "iceberg-spark-4.1_2.13"
project(":iceberg-spark:spark-extensions-4.1_2.13").projectDir = file('spark/v4.1/spark-extensions')
project(":iceberg-spark:spark-extensions-4.1_2.13").name = "iceberg-spark-extensions-4.1_2.13"
project(":iceberg-spark:spark-runtime-4.1_2.13").projectDir = file('spark/v4.1/spark-runtime')
project(":iceberg-spark:spark-runtime-4.1_2.13").name = "iceberg-spark-runtime-4.1_2.13"
include ":iceberg-spark:spark-4.1_2.13"
include ":iceberg-spark:spark-extensions-4.1_2.13"
include ":iceberg-spark:spark-runtime-4.1_2.13"
project(":iceberg-spark:spark-4.1_2.13").projectDir = file('spark/v4.1/spark')
project(":iceberg-spark:spark-4.1_2.13").name = "iceberg-spark-4.1_2.13"
project(":iceberg-spark:spark-extensions-4.1_2.13").projectDir = file('spark/v4.1/spark-extensions')
project(":iceberg-spark:spark-extensions-4.1_2.13").name = "iceberg-spark-extensions-4.1_2.13"
project(":iceberg-spark:spark-runtime-4.1_2.13").projectDir = file('spark/v4.1/spark-runtime')
project(":iceberg-spark:spark-runtime-4.1_2.13").name = "iceberg-spark-runtime-4.1_2.13"
}

if (sparkVersions.contains("4.2")) {
include ":iceberg-spark:spark-4.2_2.13"
include ":iceberg-spark:spark-extensions-4.2_2.13"
include ":iceberg-spark:spark-runtime-4.2_2.13"
project(":iceberg-spark:spark-4.2_2.13").projectDir = file('spark/v4.2/spark')
project(":iceberg-spark:spark-4.2_2.13").name = "iceberg-spark-4.2_2.13"
project(":iceberg-spark:spark-extensions-4.2_2.13").projectDir = file('spark/v4.2/spark-extensions')
project(":iceberg-spark:spark-extensions-4.2_2.13").name = "iceberg-spark-extensions-4.2_2.13"
project(":iceberg-spark:spark-runtime-4.2_2.13").projectDir = file('spark/v4.2/spark-runtime')
project(":iceberg-spark:spark-runtime-4.2_2.13").name = "iceberg-spark-runtime-4.2_2.13"
}

if (kafkaVersions.contains("3")) {
Expand Down
4 changes: 4 additions & 0 deletions spark/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,7 @@ if (sparkVersions.contains("4.0")) {
if (sparkVersions.contains("4.1")) {
apply from: file("$projectDir/v4.1/build.gradle")
}

if (sparkVersions.contains("4.2")) {
apply from: file("$projectDir/v4.2/build.gradle")
}
Loading
Loading