Update Spark dependency version to 3.2.0, Parquet to 1.12.1, Avro to …

…1.10.2.
bigdatagenomics · Oct 14, 2021 · 1cb9111 · 1cb9111
1 parent d60692f
commit 1cb9111
Show file tree

Hide file tree

Showing 20 changed files with 40 additions and 1,332 deletions.
diff --git a/adam-apis/pom.xml b/adam-apis/pom.xml
@@ -95,15 +95,9 @@
       <artifactId>scala-library</artifactId>
       <scope>provided</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
-      <!-- provided scope from parent -->
-    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-core_${scala.version.prefix}</artifactId>
-      <!-- provided scope from parent -->
     </dependency>
     <dependency>
       <groupId>org.bdgenomics.utils</groupId>

diff --git a/adam-assembly/pom.xml b/adam-assembly/pom.xml
@@ -39,13 +39,6 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>org.bdgenomics.adam</groupId>
-            <artifactId>adam-shade-spark3_${scala.version.prefix}</artifactId>
-            <version>${project.version}</version>
-          </dependency>
-        </dependencies>
         <configuration>
           <createDependencyReducedPom>false</createDependencyReducedPom>
           <filters>
@@ -58,12 +51,6 @@
               </excludes>
             </filter>
           </filters>
-          <relocations>
-            <relocation>
-              <pattern>org.apache.parquet.avro</pattern>
-              <shadedPattern>org.bdgenomics.adam.shaded.org.apache.parquet.avro</shadedPattern>
-            </relocation>
-          </relocations>
         </configuration>
         <executions>
           <execution>
@@ -72,7 +59,6 @@
               <goal>shade</goal>
             </goals>
             <configuration>
-              <shaderHint>workaround</shaderHint>
               <transformers>
                 <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
               </transformers>

diff --git a/adam-cli/pom.xml b/adam-cli/pom.xml
@@ -112,15 +112,9 @@
     </plugins>
   </build>
   <dependencies>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
-      <!-- provided scope from parent -->
-    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-core_${scala.version.prefix}</artifactId>
-      <!-- provided scope from parent -->
     </dependency>
     <dependency>
       <groupId>org.bdgenomics.utils</groupId>

diff --git a/adam-cli/src/main/java-templates/org/bdgenomics/adam/cli/About.java b/adam-cli/src/main/java-templates/org/bdgenomics/adam/cli/About.java
@@ -24,7 +24,6 @@ public final class About {
     private static final String ARTIFACT_ID = "${project.artifactId}";
     private static final String BUILD_TIMESTAMP = "${build-helper-maven-plugin.build.timestamp}";
     private static final String COMMIT = "${git.commit.id}";
-    private static final String HADOOP_VERSION = "${hadoop.version}";
     private static final String SCALA_VERSION = "${scala.version}";
     private static final String SPARK_VERSION = "${spark.version}";
     private static final String VERSION = "${project.version}";
@@ -56,15 +55,6 @@ public String commit() {
         return COMMIT;
     }
 
-    /**
-     * Return the Hadoop compile scope dependency version.
-     *
-     * @return the Hadoop compile scope dependency version
-     */
-    public String hadoopVersion() {
-        return HADOOP_VERSION;
-    }
-
     /**
      * Return the Scala compile scope dependency version.
      *

diff --git a/adam-cli/src/main/scala/org/bdgenomics/adam/cli/ADAMMain.scala b/adam-cli/src/main/scala/org/bdgenomics/adam/cli/ADAMMain.scala
@@ -89,8 +89,8 @@ class ADAMMain @Inject() (commandGroups: List[CommandGroup]) extends Logging {
     if (about.isSnapshot) {
       println("Commit: %s Build: %s".format(about.commit, about.buildTimestamp))
     }
-    println("Built for: Apache Spark %s, Scala %s, and Hadoop %s"
-      .format(about.sparkVersion, about.scalaVersion, about.hadoopVersion))
+    println("Built for: Apache Spark %s and Scala %s"
+      .format(about.sparkVersion, about.scalaVersion))
     println("Cite: doi:10.1145/2723372.2742787")
   }
 

diff --git a/adam-cli/src/main/scala/org/bdgenomics/adam/cli/PrintADAM.scala b/adam-cli/src/main/scala/org/bdgenomics/adam/cli/PrintADAM.scala
@@ -83,9 +83,6 @@ class PrintADAM(protected val args: PrintADAMArgs) extends BDGSparkCommand[Print
               val schema = hd.getSchema
               val writer = new GenericDatumWriter[Object](schema)
               val encoder = EncoderFactory.get().jsonEncoder(schema, out)
-              val jg = new org.codehaus.jackson.JsonFactory().createJsonGenerator(out)
-              jg.useDefaultPrettyPrinter()
-              encoder.configure(jg)
               it.foreach(pileup => {
                 writer.write(pileup, encoder)
               })

diff --git a/adam-cli/src/test/scala/org/bdgenomics/adam/cli/AboutSuite.scala b/adam-cli/src/test/scala/org/bdgenomics/adam/cli/AboutSuite.scala
@@ -25,7 +25,6 @@ class AboutSuite extends FunSuite {
   test("template variables have been replaced") {
     assert(about.artifactId !== "${project.artifactId}")
     assert(about.buildTimestamp !== "${maven.build.timestamp}")
-    assert(about.hadoopVersion !== "${hadoop.version}")
     assert(about.scalaVersion !== "${scala.version}")
     assert(about.sparkVersion !== "${spark.version}")
     assert(about.version !== "${version}")
@@ -34,7 +33,6 @@ class AboutSuite extends FunSuite {
   test("templated values are not empty") {
     assert(about.artifactId.nonEmpty)
     assert(about.buildTimestamp.nonEmpty)
-    assert(about.hadoopVersion.nonEmpty)
     assert(about.scalaVersion.nonEmpty)
     assert(about.sparkVersion.nonEmpty)
     assert(about.version.nonEmpty)

diff --git a/adam-core/pom.xml b/adam-core/pom.xml
@@ -204,15 +204,9 @@
       <artifactId>scala-library</artifactId>
       <scope>provided</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
-      <!-- provided scope from parent -->
-    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-core_${scala.version.prefix}</artifactId>
-      <!-- provided scope from parent -->
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>