Update Spark version to 2.4.3, add move to Scala 2.12 script.

bigdatagenomics · May 11, 2019 · 9841310 · 9841310
1 parent 6e6ac8d
commit 9841310
Show file tree

Hide file tree

Showing 37 changed files with 1,562 additions and 320 deletions.
diff --git a/adam-assembly/pom.xml b/adam-assembly/pom.xml
@@ -39,6 +39,13 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
+        <dependencies>
+          <dependency>
+            <groupId>org.bdgenomics.adam</groupId>
+            <artifactId>adam-shade-spark2_${scala.version.prefix}</artifactId>
+            <version>${project.version}</version>
+          </dependency>
+        </dependencies>
         <configuration>
           <createDependencyReducedPom>false</createDependencyReducedPom>
           <filters>
@@ -51,6 +58,12 @@
               </excludes>
             </filter>
           </filters>
+          <relocations>
+            <relocation>
+              <pattern>org.apache.parquet.avro</pattern>
+              <shadedPattern>org.bdgenomics.adam.shaded.org.apache.parquet.avro</shadedPattern>
+            </relocation>
+          </relocations>
         </configuration>
         <executions>
           <execution>
@@ -59,6 +72,7 @@
               <goal>shade</goal>
             </goals>
             <configuration>
+              <shaderHint>workaround</shaderHint>
               <transformers>
                 <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
               </transformers>

diff --git a/adam-cli/src/main/scala/org/bdgenomics/adam/cli/ADAM2Fasta.scala b/adam-cli/src/main/scala/org/bdgenomics/adam/cli/ADAM2Fasta.scala
@@ -19,8 +19,10 @@ package org.bdgenomics.adam.cli
 
 import grizzled.slf4j.Logging
 import org.apache.spark.SparkContext
+import org.apache.spark.rdd.RDD
 import org.bdgenomics.adam.cli.FileSystemUtils._
 import org.bdgenomics.adam.rdd.ADAMContext._
+import org.bdgenomics.formats.avro.NucleotideContigFragment
 import org.bdgenomics.utils.cli._
 import org.kohsuke.args4j.{ Argument, Option => Args4jOption }
 
@@ -63,9 +65,9 @@ class ADAM2Fasta(val args: ADAM2FastaArgs) extends BDGSparkCommand[ADAM2FastaArg
 
     val cc = if (args.coalesce > 0) {
       if (args.coalesce > contigs.rdd.partitions.length || args.forceShuffle) {
-        contigs.transform(_.coalesce(args.coalesce, shuffle = true))
+        contigs.transform((rdd: RDD[NucleotideContigFragment]) => rdd.coalesce(args.coalesce, shuffle = true))
       } else {
-        contigs.transform(_.coalesce(args.coalesce, shuffle = false))
+        contigs.transform((rdd: RDD[NucleotideContigFragment]) => rdd.coalesce(args.coalesce, shuffle = false))
       }
     } else {
       contigs

diff --git a/adam-cli/src/main/scala/org/bdgenomics/adam/cli/ADAM2Fastq.scala b/adam-cli/src/main/scala/org/bdgenomics/adam/cli/ADAM2Fastq.scala
@@ -19,10 +19,12 @@ package org.bdgenomics.adam.cli
 
 import htsjdk.samtools.ValidationStringency
 import org.apache.spark.SparkContext
+import org.apache.spark.rdd.RDD
 import org.apache.spark.storage.StorageLevel
 import org.bdgenomics.adam.cli.FileSystemUtils._
 import org.bdgenomics.adam.projections.{ AlignmentRecordField, Projection }
 import org.bdgenomics.adam.rdd.ADAMContext._
+import org.bdgenomics.formats.avro.AlignmentRecord
 import org.bdgenomics.utils.cli._
 import org.kohsuke.args4j.{ Argument, Option => Args4jOption }
 
@@ -81,7 +83,7 @@ class ADAM2Fastq(val args: ADAM2FastqArgs) extends BDGSparkCommand[ADAM2FastqArg
 
     if (args.repartition != -1) {
       info("Repartitioning reads to to '%d' partitions".format(args.repartition))
-      reads = reads.transform(_.repartition(args.repartition))
+      reads = reads.transform((rdd: RDD[AlignmentRecord]) => rdd.repartition(args.repartition))
     }
 
     reads.saveAsFastq(

diff --git a/adam-cli/src/main/scala/org/bdgenomics/adam/cli/CountReadKmers.scala b/adam-cli/src/main/scala/org/bdgenomics/adam/cli/CountReadKmers.scala
@@ -19,9 +19,11 @@ package org.bdgenomics.adam.cli
 
 import grizzled.slf4j.Logging
 import org.apache.spark.SparkContext
+import org.apache.spark.rdd.RDD
 import org.bdgenomics.adam.cli.FileSystemUtils._
 import org.bdgenomics.adam.projections.{ AlignmentRecordField, Projection }
 import org.bdgenomics.adam.rdd.ADAMContext._
+import org.bdgenomics.formats.avro.AlignmentRecord
 import org.bdgenomics.utils.cli._
 import org.kohsuke.args4j.{ Argument, Option => Args4jOption }
 
@@ -61,7 +63,7 @@ class CountReadKmers(protected val args: CountReadKmersArgs) extends BDGSparkCom
 
     if (args.repartition != -1) {
       info("Repartitioning reads to '%d' partitions".format(args.repartition))
-      adamRecords = adamRecords.transform(_.repartition(args.repartition))
+      adamRecords = adamRecords.transform((rdd: RDD[AlignmentRecord]) => rdd.repartition(args.repartition))
     }
 
     // count kmers

diff --git a/adam-cli/src/main/scala/org/bdgenomics/adam/cli/Fasta2ADAM.scala b/adam-cli/src/main/scala/org/bdgenomics/adam/cli/Fasta2ADAM.scala
@@ -19,8 +19,10 @@ package org.bdgenomics.adam.cli
 
 import grizzled.slf4j.Logging
 import org.apache.spark.SparkContext
+import org.apache.spark.rdd.RDD
 import org.bdgenomics.adam.cli.FileSystemUtils._
 import org.bdgenomics.adam.rdd.ADAMContext._
+import org.bdgenomics.formats.avro.NucleotideContigFragment
 import org.bdgenomics.utils.cli._
 import org.kohsuke.args4j.{ Argument, Option => Args4jOption }
 
@@ -63,7 +65,7 @@ class Fasta2ADAM(protected val args: Fasta2ADAMArgs) extends BDGSparkCommand[Fas
 
     info("Writing records to disk.")
     val finalFasta = if (args.partitions > 0) {
-      adamFasta.transform(_.repartition(args.partitions))
+      adamFasta.transform((rdd: RDD[NucleotideContigFragment]) => rdd.repartition(args.partitions))
     } else {
       adamFasta
     }

diff --git a/adam-cli/src/main/scala/org/bdgenomics/adam/cli/Reads2Coverage.scala b/adam-cli/src/main/scala/org/bdgenomics/adam/cli/Reads2Coverage.scala
@@ -18,11 +18,13 @@
 package org.bdgenomics.adam.cli
 
 import org.apache.spark.SparkContext
+import org.apache.spark.rdd.RDD
 import org.bdgenomics.adam.cli.FileSystemUtils._
 import org.bdgenomics.adam.projections.AlignmentRecordField._
 import org.bdgenomics.adam.projections.Projection
 import org.bdgenomics.adam.rdd.ADAMContext._
 import org.bdgenomics.adam.rdd.read.AlignmentRecordDataset
+import org.bdgenomics.formats.avro.AlignmentRecord
 import org.bdgenomics.utils.cli._
 import org.kohsuke.args4j.{ Argument, Option => Args4jOption }
 
@@ -82,9 +84,9 @@ class Reads2Coverage(protected val args: Reads2CoverageArgs) extends BDGSparkCom
     val readsRdd: AlignmentRecordDataset = sc.loadAlignments(args.inputPath)
 
     val finalReads = if (args.onlyNegativeStrands && !args.onlyPositiveStrands) {
-      readsRdd.transform(rdd => rdd.filter(_.getReadNegativeStrand))
+      readsRdd.transform((rdd: RDD[AlignmentRecord]) => rdd.filter(_.getReadNegativeStrand))
     } else if (!args.onlyNegativeStrands && args.onlyPositiveStrands) {
-      readsRdd.transform(rdd => rdd.filter(!_.getReadNegativeStrand))
+      readsRdd.transform((rdd: RDD[AlignmentRecord]) => rdd.filter(!_.getReadNegativeStrand))
     } else {
       readsRdd
     }