From f93b20b4d4ef967a56ad7a7c405579e542baad2b Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Fri, 9 Jun 2017 02:52:15 +0000 Subject: [PATCH] 0.23.1 --- build.sbt | 4 +-- project/plugins.sbt | 2 +- .../adam/rdd/ADAMContextSuite.scala | 2 +- .../adam/rdd/ParallelFileMergerSuite.scala | 26 ++++++++++--------- .../adam/rdd/fragment/FragmentRDDSuite.scala | 2 +- .../rdd/read/AlignmentRecordRDDSuite.scala | 4 +-- .../bdgenomics/adam/util/ADAMFunSuite.scala | 5 +++- 7 files changed, 25 insertions(+), 20 deletions(-) diff --git a/build.sbt b/build.sbt index cb5f28f26a..061a875742 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ organization := "org.hammerlab.adam" name := sparkName("core") -version := "0.23.1-SNAPSHOT" +version := "0.23.1" addSparkDeps publishTestJar @@ -20,7 +20,7 @@ deps ++= Seq( libs.value('bdg_utils_metrics), libs.value('bdg_utils_misc), libs.value('commons_io), - libs.value('hadoop_bam), + "org.seqdoop" % "hadoop-bam" % "7.8.0" exclude("org.apache.hadoop", "hadoop-client"), libs.value('htsjdk), libs.value('loci), libs.value('log4j), diff --git a/project/plugins.sbt b/project/plugins.sbt index cf38390554..1dd7897fde 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1 +1 @@ -addSbtPlugin("org.hammerlab" % "sbt-parent" % "1.7.7-SNAPSHOT") +addSbtPlugin("org.hammerlab" % "sbt-parent" % "2.0.1") diff --git a/src/test/scala/org/bdgenomics/adam/rdd/ADAMContextSuite.scala b/src/test/scala/org/bdgenomics/adam/rdd/ADAMContextSuite.scala index 94073c2168..780d7fd89d 100644 --- a/src/test/scala/org/bdgenomics/adam/rdd/ADAMContextSuite.scala +++ b/src/test/scala/org/bdgenomics/adam/rdd/ADAMContextSuite.scala @@ -91,7 +91,7 @@ class ADAMContextSuite test("can read a small .CRAM file") { val path = testFile("artificial.cram") val referencePath = resourceUrl("artificial.fa").toString - sc.hadoopConfiguration.set(REFERENCE_SOURCE_PATH_PROPERTY, + hadoopConf.set(REFERENCE_SOURCE_PATH_PROPERTY, referencePath) val reads: RDD[AlignmentRecord] = sc.loadAlignments(path).rdd reads.count() should === (10) diff --git a/src/test/scala/org/bdgenomics/adam/rdd/ParallelFileMergerSuite.scala b/src/test/scala/org/bdgenomics/adam/rdd/ParallelFileMergerSuite.scala index 167ebb6847..89cd7321f6 100644 --- a/src/test/scala/org/bdgenomics/adam/rdd/ParallelFileMergerSuite.scala +++ b/src/test/scala/org/bdgenomics/adam/rdd/ParallelFileMergerSuite.scala @@ -21,7 +21,7 @@ import org.apache.hadoop.fs.{ FileSystem, Path } import org.bdgenomics.adam.rdd.ADAMContext._ import org.bdgenomics.adam.rdd.ParallelFileMerger._ import org.bdgenomics.adam.util.ADAMFunSuite -import org.seqdoop.hadoop_bam.CRAMInputFormat +import org.seqdoop.hadoop_bam.CRAMInputFormat.REFERENCE_SOURCE_PATH_PROPERTY class ParallelFileMergerSuite extends ADAMFunSuite { @@ -63,7 +63,7 @@ class ParallelFileMergerSuite val fileSizes = Seq(29408, 3093) - val fs = FileSystem.get(sc.hadoopConfiguration) + val fs = FileSystem.get(hadoopConf) val (size, sizes) = getFullSize(fs, files) assert(size === fileSizes.sum.toLong) @@ -93,7 +93,7 @@ class ParallelFileMergerSuite ) .map(new Path(_)) - val fs = FileSystem.get(sc.hadoopConfiguration) + val fs = FileSystem.get(hadoopConf) val fileSizesMap = files .map(f ⇒ (f, fs.getFileStatus(f).getLen().toInt)) @@ -127,7 +127,7 @@ class ParallelFileMergerSuite ) .map(new Path(_)) - val fs = FileSystem.get(sc.hadoopConfiguration) + val fs = FileSystem.get(hadoopConf) val fileSizesMap = files .map(f ⇒ (f, fs.getFileStatus(f).getLen().toInt)) @@ -173,7 +173,7 @@ class ParallelFileMergerSuite reads.transform(_.repartition(4)) .saveAsSam(outPath, asSingleFile = true, deferMerging = true) - val fs = FileSystem.get(sc.hadoopConfiguration) + val fs = FileSystem.get(hadoopConf) val filesToMerge = (Seq(outPath + "_head") ++ (0 until 4).map(i => { (outPath + "_tail") / "part-r-0000%d".format(i) })).map(new Path(_)) @@ -182,7 +182,7 @@ class ParallelFileMergerSuite mergePaths( outPath, filesToMerge, - sc.broadcast(sc.hadoopConfiguration), + sc.broadcast(hadoopConf), false, false ) @@ -199,7 +199,7 @@ class ParallelFileMergerSuite reads.transform(_.repartition(4)) .saveAsSam(outPath, asSingleFile = true, deferMerging = true) - val fs = FileSystem.get(sc.hadoopConfiguration) + val fs = FileSystem.get(hadoopConf) val filesToMerge = (Seq(outPath + "_head") ++ (0 until 4).map(i => { (outPath + "_tail") / "part-r-0000%d".format(i) })).map(new Path(_)) @@ -208,7 +208,7 @@ class ParallelFileMergerSuite mergePaths( outPath, filesToMerge, - sc.broadcast(sc.hadoopConfiguration), + sc.broadcast(hadoopConf), true, false ) @@ -220,15 +220,17 @@ class ParallelFileMergerSuite test("merge a sharded cram file") { val referencePath = resourceUrl("artificial.fa").toString - sc.hadoopConfiguration.set(CRAMInputFormat.REFERENCE_SOURCE_PATH_PROPERTY, - referencePath) + hadoopConf.set( + REFERENCE_SOURCE_PATH_PROPERTY, + referencePath + ) val reads = sc.loadAlignments(testFile("artificial.cram")) val outPath = tmpFile("out.cram") reads.transform(_.repartition(4)) .saveAsSam(outPath, isSorted = true, asSingleFile = true, deferMerging = true) - val fs = FileSystem.get(sc.hadoopConfiguration) + val fs = FileSystem.get(hadoopConf) val filesToMerge = (Seq(outPath + "_head") ++ (0 until 4).map(i => { (outPath + "_tail") / "part-r-0000%d".format(i) })).map(new Path(_)) @@ -237,7 +239,7 @@ class ParallelFileMergerSuite mergePaths( outPath, filesToMerge, - sc.broadcast(sc.hadoopConfiguration), + sc.broadcast(hadoopConf), false, true ) diff --git a/src/test/scala/org/bdgenomics/adam/rdd/fragment/FragmentRDDSuite.scala b/src/test/scala/org/bdgenomics/adam/rdd/fragment/FragmentRDDSuite.scala index 48bacb0048..2c9cd8fc87 100644 --- a/src/test/scala/org/bdgenomics/adam/rdd/fragment/FragmentRDDSuite.scala +++ b/src/test/scala/org/bdgenomics/adam/rdd/fragment/FragmentRDDSuite.scala @@ -25,7 +25,7 @@ class FragmentRDDSuite extends ADAMFunSuite { test("don't lose any reads when piping interleaved fastq to sam") { // write suffixes at end of reads - sc.hadoopConfiguration.setBoolean(InterleavedFASTQInFormatter.WRITE_SUFFIXES, true) + hadoopConf.setBoolean(InterleavedFASTQInFormatter.WRITE_SUFFIXES, true) val fragmentsPath = testFile("interleaved_fastq_sample1.ifq") val ardd = sc.loadFragments(fragmentsPath) diff --git a/src/test/scala/org/bdgenomics/adam/rdd/read/AlignmentRecordRDDSuite.scala b/src/test/scala/org/bdgenomics/adam/rdd/read/AlignmentRecordRDDSuite.scala index ca65a1f509..6333b31a30 100644 --- a/src/test/scala/org/bdgenomics/adam/rdd/read/AlignmentRecordRDDSuite.scala +++ b/src/test/scala/org/bdgenomics/adam/rdd/read/AlignmentRecordRDDSuite.scala @@ -217,7 +217,7 @@ class AlignmentRecordRDDSuite test("round trip with single CRAM file produces equivalent Read values") { val readsPath = testFile("artificial.cram") val referencePath = resourceUrl("artificial.fa").toString - sc.hadoopConfiguration.set(REFERENCE_SOURCE_PATH_PROPERTY, + hadoopConf.set(REFERENCE_SOURCE_PATH_PROPERTY, referencePath) val ardd = sc.loadBam(readsPath) @@ -250,7 +250,7 @@ class AlignmentRecordRDDSuite test("round trip with sharded CRAM file produces equivalent Read values") { val readsPath = testFile("artificial.cram") val referencePath = resourceUrl("artificial.fa").toString - sc.hadoopConfiguration.set(REFERENCE_SOURCE_PATH_PROPERTY, + hadoopConf.set(REFERENCE_SOURCE_PATH_PROPERTY, referencePath) val ardd = sc.loadBam(readsPath) diff --git a/src/test/scala/org/bdgenomics/adam/util/ADAMFunSuite.scala b/src/test/scala/org/bdgenomics/adam/util/ADAMFunSuite.scala index 99eef4c802..6fedf21c32 100644 --- a/src/test/scala/org/bdgenomics/adam/util/ADAMFunSuite.scala +++ b/src/test/scala/org/bdgenomics/adam/util/ADAMFunSuite.scala @@ -29,7 +29,10 @@ import org.hammerlab.test.resources.{ File, Url } import org.scalactic.TypeCheckedTripleEquals abstract class ADAMFunSuite - extends KryoSparkSuite(classOf[ADAMKryoRegistrator], referenceTracking = true) + extends KryoSparkSuite( + classOf[ADAMKryoRegistrator], + referenceTracking = true + ) with ContigNameCanEqualString with LocusCanEqualInt with ClearContigNames