Merge pull request #410 from Acxiom/spark_3.3.2_migration

Update GCP drivers to ensure Metalus works properly with GCP Storage
Acxiom · May 16, 2024 · d16503a · d16503a
2 parents 418e8d7 + 9004011
commit d16503a
Show file tree

Hide file tree

Showing 12 changed files with 152 additions and 148 deletions.
diff --git a/metalus-application/pom.xml b/metalus-application/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.9.7-SNAPSHOT</version>
+        <version>1.9.8-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>

diff --git a/metalus-aws/pom.xml b/metalus-aws/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.9.7-SNAPSHOT</version>
+        <version>1.9.8-SNAPSHOT</version>
     </parent>
 
     <properties>

diff --git a/metalus-common/pom.xml b/metalus-common/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.9.7-SNAPSHOT</version>
+        <version>1.9.8-SNAPSHOT</version>
     </parent>
 
     <dependencies>

diff --git a/metalus-core/pom.xml b/metalus-core/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.9.7-SNAPSHOT</version>
+        <version>1.9.8-SNAPSHOT</version>
     </parent>
 
     <properties>

diff --git a/metalus-delta/pom.xml b/metalus-delta/pom.xml
@@ -12,7 +12,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.9.7-SNAPSHOT</version>
+        <version>1.9.8-SNAPSHOT</version>
     </parent>
 
     <dependencies>

diff --git a/metalus-examples/pom.xml b/metalus-examples/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.9.7-SNAPSHOT</version>
+        <version>1.9.8-SNAPSHOT</version>
     </parent>
 
     <properties>

diff --git a/metalus-gcp/pom.xml b/metalus-gcp/pom.xml
@@ -9,15 +9,15 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.9.7-SNAPSHOT</version>
+        <version>1.9.8-SNAPSHOT</version>
     </parent>
 
     <dependencyManagement>
         <dependencies>
             <dependency>
                 <groupId>com.google.cloud</groupId>
                 <artifactId>libraries-bom</artifactId>
-                <version>5.7.0</version>
+                <version>26.37.0</version>
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
@@ -45,11 +45,11 @@
             <scope>provided</scope>
         </dependency>
         <!-- Google Libraries -->
-        <dependency>
-            <groupId>io.grpc</groupId>
-            <artifactId>grpc-netty-shaded</artifactId>
-            <version>1.39.0</version>
-        </dependency>
+<!--        <dependency>-->
+<!--            <groupId>io.grpc</groupId>-->
+<!--            <artifactId>grpc-netty-shaded</artifactId>-->
+<!--            <version>1.39.0</version>-->
+<!--        </dependency>-->
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
@@ -66,11 +66,11 @@
             <artifactId>google-cloud-datastore</artifactId>
             <version>1.107.1</version>
         </dependency>
-        <dependency>
-            <groupId>io.grpc</groupId>
-            <artifactId>grpc-core</artifactId>
-            <version>1.39.0</version>
-        </dependency>
+<!--        <dependency>-->
+<!--            <groupId>io.grpc</groupId>-->
+<!--            <artifactId>grpc-core</artifactId>-->
+<!--            <version>1.39.0</version>-->
+<!--        </dependency>-->
         <dependency>
             <groupId>io.opencensus</groupId>
             <artifactId>opencensus-api</artifactId>
@@ -85,28 +85,28 @@
             <groupId>com.google.cloud</groupId>
             <artifactId>google-cloud-pubsub</artifactId>
             <version>1.128.0</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>io.grpc</groupId>
-                    <artifactId>grpc-netty-shaded</artifactId>
-                </exclusion>
-            </exclusions>
+<!--            <exclusions>-->
+<!--                <exclusion>-->
+<!--                    <groupId>io.grpc</groupId>-->
+<!--                    <artifactId>grpc-netty-shaded</artifactId>-->
+<!--                </exclusion>-->
+<!--            </exclusions>-->
         </dependency>
         <dependency>
             <groupId>com.google.cloud.bigdataoss</groupId>
             <artifactId>gcs-connector</artifactId>
-            <version>hadoop3-2.2.21</version>
+            <version>hadoop2-2.2.21</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>com.google.cloud</groupId>
             <artifactId>google-cloud-storage</artifactId>
-            <version>1.118.1</version>
+            <version>2.37.0</version>
         </dependency>
         <dependency>
             <groupId>com.google.cloud</groupId>
             <artifactId>google-cloud-secretmanager</artifactId>
-            <version>2.40.0</version>
+            <version>1.1.0</version>
             <exclusions>
                 <exclusion>
                     <groupId>io.grpc</groupId>
@@ -117,7 +117,7 @@
         <dependency>
             <groupId>com.google.cloud</groupId>
             <artifactId>google-cloud-nio</artifactId>
-            <version>0.127.8</version>
+            <version>0.121.0</version>
         </dependency>
     </dependencies>
 
@@ -162,6 +162,9 @@
                             </excludes>
                         </filter>
                     </filters>
+                    <transformers>
+                        <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+                    </transformers>
                 </configuration>
                 <executions>
                     <execution>
@@ -178,6 +181,7 @@
                                         <exclude>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem</exclude>
                                         <exclude>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS</exclude>
                                         <exclude>com.google.cloud.spark.bigquery.DefaultSource</exclude>
+                                        <exclude>io.grpc.CallOptions.getOnReadyThreshold</exclude>
                                     </excludes>
                                 </relocation>
                             </relocations>