From a4f6242eaee43ef5c2c7b926d90cd46bb23c963d Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Thu, 14 May 2026 11:33:06 -0400 Subject: [PATCH 1/2] pin hadoop to 3.3.6 --- .../trigger_files/IO_Iceberg_Integration_Tests.json | 2 +- sdks/java/io/iceberg/build.gradle | 10 +++++++++- .../apache/beam/sdk/io/iceberg/IcebergScanConfig.java | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests.json b/.github/trigger_files/IO_Iceberg_Integration_Tests.json index b73af5e61a43..7ab7bcd9a9c6 100644 --- a/.github/trigger_files/IO_Iceberg_Integration_Tests.json +++ b/.github/trigger_files/IO_Iceberg_Integration_Tests.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run.", - "modification": 1 + "modification": 2 } diff --git a/sdks/java/io/iceberg/build.gradle b/sdks/java/io/iceberg/build.gradle index bbd55fee2fc8..5c67398f9c71 100644 --- a/sdks/java/io/iceberg/build.gradle +++ b/sdks/java/io/iceberg/build.gradle @@ -59,7 +59,7 @@ dependencies { implementation "org.apache.iceberg:iceberg-parquet:$iceberg_version" implementation "org.apache.iceberg:iceberg-orc:$iceberg_version" implementation "org.apache.iceberg:iceberg-data:$iceberg_version" - implementation library.java.hadoop_common + implementation "org.apache.hadoop:hadoop-common:3.3.6" // TODO(https://github.com/apache/beam/issues/21156): Determine how to build without this dependency provided "org.immutables:value:2.8.8" permitUnusedDeclared "org.immutables:value:2.8.8" @@ -117,6 +117,14 @@ dependencies { configurations.all { // iceberg-core needs avro:1.12.0 resolutionStrategy.force 'org.apache.avro:avro:1.12.0' + // bigdataoss:gcs-connector and parquet:parquet-hadoop have conflicts with global hadoop-common:3.4.2 + // upgrading gcs-connector to 4.0.0 would be fine, because it uses hadoop-common 3.4.2 + // but parquet-hadoop is still at 3.3.0 + // so for now we need to pin hadoop to 3.3.6 until parquet-hadoop releases a version that uses hadoop 3.4.2+ + resolutionStrategy.force 'org.apache.hadoop:hadoop-common:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-client:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs-client:3.3.6' } hadoopVersions.each {kv -> diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java index 56f21d2bc8b5..45ecc7cf71c3 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java @@ -19,7 +19,7 @@ import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.hadoop.util.Sets.newHashSet; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets.newHashSet; import com.google.auto.value.AutoValue; import java.io.Serializable; From 3558a210c7357a7845af5c2556b67bd7d2d23db6 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Thu, 14 May 2026 12:51:33 -0400 Subject: [PATCH 2/2] pin in expansion service too --- .../IO_Iceberg_Integration_Tests_Dataflow.json | 2 +- sdks/java/io/expansion-service/build.gradle | 10 ++++++++++ sdks/java/io/iceberg/build.gradle | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json b/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json index 5abe02fc09c7..3a009261f4f9 100644 --- a/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json +++ b/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run.", - "modification": 1 + "modification": 2 } diff --git a/sdks/java/io/expansion-service/build.gradle b/sdks/java/io/expansion-service/build.gradle index 4e4c15f367cb..2932e02e38c9 100644 --- a/sdks/java/io/expansion-service/build.gradle +++ b/sdks/java/io/expansion-service/build.gradle @@ -57,6 +57,16 @@ configurations.runtimeClasspath { // Pin nimbus-jose-jwt to 9.37.4 to fix CVE-2025-53864 (transitive via hadoop-auth) resolutionStrategy.force 'com.nimbusds:nimbus-jose-jwt:9.37.4' + + // [iceberg] + // bigdataoss:gcs-connector and parquet:parquet-hadoop have conflicts with global hadoop-common:3.4.2 + // upgrading gcs-connector to 4.0.0 would be fine, because it uses hadoop-common 3.4.2 + // but parquet-hadoop is still at 3.3.0 + // so for now we need to pin hadoop to 3.3.6 until parquet-hadoop releases a version that uses hadoop 3.4.2+ + resolutionStrategy.force 'org.apache.hadoop:hadoop-common:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-client:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs-client:3.3.6' } shadowJar { diff --git a/sdks/java/io/iceberg/build.gradle b/sdks/java/io/iceberg/build.gradle index 5c67398f9c71..2dae0335f9c9 100644 --- a/sdks/java/io/iceberg/build.gradle +++ b/sdks/java/io/iceberg/build.gradle @@ -70,6 +70,7 @@ dependencies { runtimeOnly "org.apache.iceberg:iceberg-azure:$iceberg_version" runtimeOnly "org.apache.iceberg:iceberg-azure-bundle:$iceberg_version" runtimeOnly library.java.bigdataoss_gcs_connector + runtimeOnly library.java.bigdataoss_util_hadoop runtimeOnly library.java.hadoop_client testImplementation project(":sdks:java:managed")