From e4a539ca3ea2b8f7f4b9bdef243e7498347bb4bf Mon Sep 17 00:00:00 2001 From: Frank Austin Nothaft Date: Wed, 24 Aug 2016 08:32:19 -0700 Subject: [PATCH] [ADAM-1093] Move to support Spark 2.0.0. Relies on https://github.com/bigdatagenomics/utils/pull/78. Resolves #1093: * Clean up logging in VariantContextRDD * Add move_to_spark_2.sh script and CI hooks --- .../bdgenomics/adam/rdd/variation/VariantContextRDD.scala | 2 +- pom.xml | 2 +- scripts/jenkins-test | 7 +++++++ scripts/move_to_spark_2.sh | 8 ++++++++ 4 files changed, 17 insertions(+), 2 deletions(-) create mode 100755 scripts/move_to_spark_2.sh diff --git a/adam-core/src/main/scala/org/bdgenomics/adam/rdd/variation/VariantContextRDD.scala b/adam-core/src/main/scala/org/bdgenomics/adam/rdd/variation/VariantContextRDD.scala index a525891846..11c29da4f3 100644 --- a/adam-core/src/main/scala/org/bdgenomics/adam/rdd/variation/VariantContextRDD.scala +++ b/adam-core/src/main/scala/org/bdgenomics/adam/rdd/variation/VariantContextRDD.scala @@ -18,7 +18,6 @@ package org.bdgenomics.adam.rdd.variation import org.apache.hadoop.io.LongWritable -import org.apache.spark.Logging import org.apache.spark.rdd.RDD import org.bdgenomics.adam.converters.VariantContextConverter import org.bdgenomics.adam.models.{ @@ -28,6 +27,7 @@ import org.bdgenomics.adam.models.{ } import org.bdgenomics.adam.rdd.MultisampleGenomicRDD import org.bdgenomics.formats.avro.Sample +import org.bdgenomics.utils.misc.Logging import org.bdgenomics.utils.cli.SaveArgs import org.seqdoop.hadoop_bam._ diff --git a/pom.xml b/pom.xml index 6306bbe26f..028db07497 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 7.6.0 1.7.21 0.9.0 - 0.2.7 + 0.2.8-SNAPSHOT 2.5.0 diff --git a/scripts/jenkins-test b/scripts/jenkins-test index edff72a826..008a4867da 100755 --- a/scripts/jenkins-test +++ b/scripts/jenkins-test @@ -28,6 +28,13 @@ then exit 1 fi +# are we testing for spark 2.0.0? if so, we need to rewrite our poms first +if [ ${SPARK_VERSION} == 2.0.0 ]; +then + echo "Rewriting POM.xml files for Scala 2.10." + ./scripts/move_to_spark_2.sh +fi + # are we testing for scala 2.11? if so, we need to rewrite our poms to 2.11 first if [ ${SCALAVER} == 2.11 ]; then diff --git a/scripts/move_to_spark_2.sh b/scripts/move_to_spark_2.sh new file mode 100755 index 0000000000..5b829e49cc --- /dev/null +++ b/scripts/move_to_spark_2.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set +x + +find . -name "pom.xml" -exec sed -e "/utils-/ s/_2.10/-spark2_2.10/g" \ + -e "/adam-/ s/_2.10/-spark2_2.10/g" \ + -e "/spark.version/ s/1.6.1/2.0.0/g" \ + -i .spark2.bak '{}' \;