[SEDONA-276][SEDONA-277] Support Spark 3.4

.github/workflows/java.yml

@@ -16,6 +16,18 @@ jobs:
       fail-fast: true
       matrix:
         include:
+          - spark: 3.4.0
+            scala: 2.13.8
+            jdk: '8'
+            skipTests: ''
+          - spark: 3.4.0
+            scala: 2.12.15
+            jdk: '8'
+            skipTests: ''
+          - spark: 3.4.0
+            scala: 2.12.15
+            jdk: '11'
+            skipTests: ''
           - spark: 3.3.0
             scala: 2.13.8
             jdk: '8'
@@ -59,7 +71,12 @@ jobs:
         SPARK_VERSION: ${{ matrix.spark }}
         SCALA_VERSION: ${{ matrix.scala }}
         SKIP_TESTS: ${{ matrix.skipTests }}
-      run: mvn -q clean install -Dscala=${SCALA_VERSION:0:4} -Dspark.version=${SPARK_VERSION} ${SKIP_TESTS}
+      run: |
+        SPARK_COMPAT_VERSION="3.0"
+        if [ ${SPARK_VERSION:2:1} -gt "3" ]; then
+          SPARK_COMPAT_VERSION=${SPARK_VERSION:0:3}
+        fi
+        mvn -q clean install -Dspark=${SPARK_COMPAT_VERSION} -Dscala=${SCALA_VERSION:0:4} -Dspark.version=${SPARK_VERSION} ${SKIP_TESTS}
     - run: mkdir staging
     - run: cp viz/target/sedona-*.jar staging
     - run: cp spark-shaded/target/sedona-*.jar staging
@@ -68,4 +85,4 @@ jobs:
     - uses: actions/upload-artifact@v2
       with:
         name: generated-jars
-        path: staging
+        path: staging

.github/workflows/python.yml

@@ -15,6 +15,22 @@ jobs:
     strategy:
       matrix:
         include:
+          - spark: '3.4.0'
+            scala: '2.12.8'
+            python: '3.10'
+            hadoop: '3'
+          - spark: '3.4.0'
+            scala: '2.12.8'
+            python: '3.9'
+            hadoop: '3'
+          - spark: '3.4.0'
+            scala: '2.12.8'
+            python: '3.8'
+            hadoop: '3'
+          - spark: '3.4.0'
+            scala: '2.12.8'
+            python: '3.7'
+            hadoop: '3'
           - spark: '3.3.0'
             scala: '2.12.8'
             python: '3.10'
@@ -61,7 +77,12 @@ jobs:
     - env:
         SPARK_VERSION: ${{ matrix.spark }}
         SCALA_VERSION: ${{ matrix.scala }}
-      run: if [ ${SPARK_VERSION:0:1} == "3" ]; then mvn -q clean install -DskipTests -Dscala=${SCALA_VERSION:0:4} -Dspark=3.0 -Dgeotools ; else mvn -q clean install -DskipTests -Dscala=${SCALA_VERSION:0:4} -Dspark=2.4 -Dgeotools ; fi
+      run: |
+          SPARK_COMPAT_VERSION="3.0"
+          if [ ${SPARK_VERSION:2:1} -gt "3" ]; then
+            SPARK_COMPAT_VERSION=${SPARK_VERSION:0:3}
+          fi
+          mvn -q clean install -DskipTests -Dspark=${SPARK_COMPAT_VERSION} -Dscala=${SCALA_VERSION:0:4} -Dgeotools
     - env:
         SPARK_VERSION: ${{ matrix.spark }}
         HADOOP_VERSION: ${{ matrix.hadoop }}

.github/workflows/r.yml

@@ -15,12 +15,14 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        spark: [3.0.3, 3.1.2, 3.2.1, 3.3.0]
+        spark: [3.0.3, 3.1.2, 3.2.1, 3.3.0, 3.4.0]
+        hadoop: [3]
         scala: [2.12.15]
         r: [oldrel, release]
 
     env:
       SPARK_VERSION: ${{ matrix.spark }}
+      HADOOP_VERSION: ${{ matrix.hadoop }}
       SCALA_VERSION: ${{ matrix.scala }}
       # Ensure the temporary auth token for this workflow, instead of the
       # bundled GitHub PAT from the `remotes` package is used for
@@ -84,13 +86,23 @@ jobs:
           key: apache.sedona-apache-spark-${{ steps.os-name.outputs.os-name }}-${{ env.SPARK_VERSION }}
       - name: Build Sedona libraries
         run: |
-          if [ ${SPARK_VERSION:0:1} == "3" ]; then
-            mvn -q clean install -DskipTests -Dscala=${SCALA_VERSION:0:4} -Dspark=3.0 -Dgeotools
-          else
-            mvn -q clean install -DskipTests -Dscala=${SCALA_VERSION:0:4} -Dspark=2.4 -Dgeotools
+          SPARK_COMPAT_VERSION="3.0"
+          if [ ${SPARK_VERSION:2:1} -gt "3" ]; then
+            SPARK_COMPAT_VERSION=${SPARK_VERSION:0:3}
           fi
+          mvn -q clean install -DskipTests -Dspark=${SPARK_COMPAT_VERSION} -Dscala=${SCALA_VERSION:0:4}
       - name: Run tests
         run: |
+          if [[ "${SPARK_VERSION:0:3}" < "3.3" ]]; then
+            case "$HADOOP_VERSION" in
+              3)
+                export HADOOP_VERSION=3.2
+                ;;
+              2)
+                export HADOOP_VERSION=2.7
+                ;;
+            esac
+          fi
           export SPARKLYR_LOG_FILE='/tmp/sparklyr.log'
           source ./.github/workflows/scripts/prepare_sparklyr_sedona_test_env.sh
           echo "Apache Sedona jar files: ${SEDONA_JAR_FILES}"

R/tests/testthat/helper-initialize.R

@@ -19,9 +19,10 @@ testthat_spark_connection <- function(conn_retry_interval_s = 2) {
   conn_key <- ".testthat_spark_connection"
   if (!exists(conn_key, envir = .GlobalEnv)) {
     version <- Sys.getenv("SPARK_VERSION")
+    hadoop_version <- Sys.getenv("HADOOP_VERSION")
     spark_installed <- spark_installed_versions()
-    if (nrow(spark_installed[spark_installed$spark == version, ]) == 0) {
-      spark_install(version)
+    if (nrow(spark_installed[spark_installed$spark == version & spark_installed$hadoop_version == hadoop_version, ]) == 0) {
+      spark_install(version, hadoop_version)
     }
 
     conn_attempts <- 3
@@ -37,7 +38,8 @@ testthat_spark_connection <- function(conn_retry_interval_s = 2) {
             method = "shell",
             config = config,
             app_name = paste0("testthat-", uuid::UUIDgenerate()),
-            version = version
+            version = version,
+            hadoop_version = hadoop_version
           )
           assign(conn_key, sc, envir = .GlobalEnv)
           TRUE

common/pom.xml

@@ -32,6 +32,10 @@
     <url>http://sedona.apache.org/</url>
     <packaging>jar</packaging>
 
+    <properties>
+        <maven.deploy.skip>${skip.deploy.common.modules}</maven.deploy.skip>
+    </properties>
+
     <dependencies>
         <dependency>
             <groupId>org.slf4j</groupId>
@@ -83,19 +87,4 @@
             </plugin>
         </plugins>
     </build>
-    <profiles>
-        <profile>
-            <id>common-scala2.13</id>
-            <activation>
-                <property>
-                    <name>scala</name>
-                    <value>2.13</value>
-                </property>
-                <activeByDefault>false</activeByDefault>
-            </activation>
-            <properties>
-                <maven.deploy.skip>true</maven.deploy.skip>
-            </properties>
-        </profile>
-    </profiles>
 </project>

core/pom.xml

@@ -52,6 +52,8 @@
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-core_${scala.compat.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
             <exclusions>
                 <!-- Make sure Hadoop's guava version is used for older versions of Spark tests -->
                 <exclusion>
@@ -67,11 +69,27 @@
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-client-runtime</artifactId>
                 </exclusion>
+                <!-- Exclude log4j 1 for older versions of Spark-->
+                <exclusion>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+                <!-- Exclude log4j-slf4j2-impl for Spark 3.4 -->
+                <exclusion>
+                    <groupId>org.apache.logging.log4j</groupId>
+                    <artifactId>log4j-slf4j2-impl</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_${scala.compat.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
@@ -122,12 +140,15 @@
         <dependency>
             <groupId>org.scala-lang</groupId>
             <artifactId>scala-library</artifactId>
+            <version>${scala.version}</version>
         </dependency>
 
         <!-- Test -->
         <dependency>
             <groupId>org.scalatest</groupId>
             <artifactId>scalatest_${scala.compat.version}</artifactId>
+            <version>${scalatest.version}</version>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>

flink-shaded/pom.xml

@@ -32,7 +32,7 @@
     <packaging>jar</packaging>
 
     <properties>
-        <maven.deploy.skip>false</maven.deploy.skip>
+        <maven.deploy.skip>${skip.deploy.common.modules}</maven.deploy.skip>
         <cdm.scope>compile</cdm.scope>
     </properties>
 
@@ -51,17 +51,6 @@
     <build>
         <sourceDirectory>src/main/scala</sourceDirectory>
         <plugins>
-            <plugin>
-                <groupId>io.paradoxical</groupId>
-                <artifactId>resolved-pom-maven-plugin</artifactId>
-                <version>1.0</version>
-                <executions>
-                    <execution>
-                        <id>resolve-my-pom</id>
-                        <phase>none</phase>
-                    </execution>
-                </executions>
-            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>

flink/pom.xml

@@ -33,7 +33,7 @@
 	<packaging>jar</packaging>
 
     <properties>
-        <maven.deploy.skip>false</maven.deploy.skip>
+        <maven.deploy.skip>${skip.deploy.common.modules}</maven.deploy.skip>
         <flink.version>1.14.3</flink.version>
         <flink.scope>provided</flink.scope>
     </properties>
@@ -129,10 +129,13 @@
         <dependency>
             <groupId>org.scala-lang</groupId>
             <artifactId>scala-library</artifactId>
+            <version>${scala.version}</version>
         </dependency>
         <dependency>
             <groupId>org.scalatest</groupId>
             <artifactId>scalatest_${scala.compat.version}</artifactId>
+            <version>${scalatest.version}</version>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>