ArrayIndexOutOfBoundsException in PhredUtils$.phredToSuccessProbability

[heuermh]

scala> val genotypes = sc.loadGenotypes("adam-core/src/test/resources/gvcf_multiallelic/multiallelic.vcf")
genotypes: org.bdgenomics.adam.rdd.variant.GenotypeRDD =
RDDBoundGenotypeRDD...

scala> genotypes.rdd.count()
17/09/07 10:34:11 ERROR Executor: Exception in task 0.0 in stage 0.0 (TID 0)
java.lang.ArrayIndexOutOfBoundsException: -2147483648
	at org.bdgenomics.adam.util.PhredUtils$.phredToSuccessProbability(PhredUtils.scala:68)
	at org.bdgenomics.adam.util.PhredUtils$.phredToLogProbability(PhredUtils.scala:56)
	at org.bdgenomics.adam.converters.VariantContextConverter$$anonfun$formatNonRefGenotypeLikelihoods$1.apply(VariantContextConverter.scala:863)
	at org.bdgenomics.adam.converters.VariantContextConverter$$anonfun$formatNonRefGenotypeLikelihoods$1.apply(VariantContextConverter.scala:862)
	at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
	at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
	at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
	at scala.collection.mutable.ArrayOps$ofInt.foreach(ArrayOps.scala:234)
	at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
	at scala.collection.mutable.ArrayOps$ofInt.map(ArrayOps.scala:234)
	at org.bdgenomics.adam.converters.VariantContextConverter.formatNonRefGenotypeLikelihoods(VariantContextConverter.scala:862)
	at org.bdgenomics.adam.converters.VariantContextConverter$$anonfun$80.apply(VariantContextConverter.scala:1691)
	at org.bdgenomics.adam.converters.VariantContextConverter$$anonfun$80.apply(VariantContextConverter.scala:1686)
	at scala.Option.fold(Option.scala:158)
	at org.bdgenomics.adam.converters.VariantContextConverter.org$bdgenomics$adam$converters$VariantContextConverter$$convert$2(VariantContextConverter.scala:1686)
	at org.bdgenomics.adam.converters.VariantContextConverter$$anonfun$makeGenotypeFormatFn$1.apply(VariantContextConverter.scala:1723)
	at org.bdgenomics.adam.converters.VariantContextConverter$$anonfun$makeGenotypeFormatFn$1.apply(VariantContextConverter.scala:1723)
	at org.bdgenomics.adam.converters.VariantContextConverter$$anonfun$5.apply(VariantContextConverter.scala:318)
	at org.bdgenomics.adam.converters.VariantContextConverter$$anonfun$5.apply(VariantContextConverter.scala:317)
	at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
	at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
	at scala.collection.Iterator$class.foreach(Iterator.scala:893)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
	at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
	at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
	at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
	at scala.collection.AbstractTraversable.map(Traversable.scala:104)
	at org.bdgenomics.adam.converters.VariantContextConverter.convert(VariantContextConverter.scala:317)

[fnothaft]

That's a bit surprising of an error to see, because we convert that test VCF into genotypes in our unit tests. Additionally, we revised the PhredUtils code to better handle very large values in 7913e76. That said, we should add a check that ensures that phred values are non-negative.

[fnothaft]

@heuermh does this repro for you on ToT?

[heuermh]

Yes.

$ ./bin/adam-submit --version
Using ADAM_MAIN=org.bdgenomics.adam.cli.ADAMMain
Using spark-submit=/usr/local/bin/spark-submit
log4j:WARN No appenders could be found for logger (org.bdgenomics.adam.cli.ADAMMain).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.

       e        888~-_         e            e    e
      d8b       888   \       d8b          d8b  d8b
     /Y88b      888    |     /Y88b        d888bdY88b
    /  Y88b     888    |    /  Y88b      / Y88Y Y888b
   /____Y88b    888   /    /____Y88b    /   YY   Y888b
  /      Y88b   888_-~    /      Y88b  /          Y888b

ADAM version: 0.24.0-SNAPSHOT
Commit: 9de489b02dc3040db1d4c20a7f56cf7358d55249 Build: 2018-01-11
Built for: Apache Spark 2.2.1, Scala 2.11.12, and Hadoop 2.7.3

$ ./bin/adam-shell 
Using SPARK_SHELL=/usr/local/bin/spark-shell
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
18/01/11 10:54:51 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Spark context Web UI available at http://192.168.1.101:4040
Spark context available as 'sc' (master = local[*], app id = local-1515689692739).
Spark session available as 'spark'.
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /___/ .__/\_,_/_/ /_/\_\   version 2.2.1
      /_/
         
Using Scala version 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_66)
Type in expressions to have them evaluated.
Type :help for more information.

scala> import org.bdgenomics.adam.rdd.ADAMContext._
import org.bdgenomics.adam.rdd.ADAMContext._

scala> val genotypes = sc.loadGenotypes("adam-core/src/test/resources/gvcf_multiallelic/multiallelic.vcf")
genotypes: org.bdgenomics.adam.rdd.variant.GenotypeRDD = RDDBoundGenotypeRDD with 86 reference sequences and 1 samples

scala> genotypes.rdd.count()
18/01/11 10:56:05 ERROR Executor: Exception in task 0.0 in stage 0.0 (TID 0)
java.lang.ArrayIndexOutOfBoundsException: -2147483648
	at org.bdgenomics.adam.util.PhredUtils$.phredToSuccessProbability(PhredUtils.scala:68)
	at org.bdgenomics.adam.util.PhredUtils$.phredToLogProbability(PhredUtils.scala:56)
	at org.bdgenomics.adam.converters.VariantContextConverter$$anonfun$formatNonRefGenotypeLikelihoods$1.apply(VariantContextConverter.scala:939)
	at org.bdgenomics.adam.converters.VariantContextConverter$$anonfun$formatNonRefGenotypeLikelihoods$1.apply(VariantContextConverter.scala:938)
	at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
	at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
	at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
	at scala.collection.mutable.ArrayOps$ofInt.foreach(ArrayOps.scala:234)
	at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
	at scala.collection.mutable.ArrayOps$ofInt.map(ArrayOps.scala:234)
	at org.bdgenomics.adam.converters.VariantContextConverter.formatNonRefGenotypeLikelihoods(VariantContextConverter.scala:938)
	at org.bdgenomics.adam.converters.VariantContextConverter$$anonfun$82.apply(VariantContextConverter.scala:1829)
	at org.bdgenomics.adam.converters.VariantContextConverter$$anonfun$82.apply(VariantContextConverter.scala:1824)
	at scala.Option.fold(Option.scala:158)
	at org.bdgenomics.adam.converters.VariantContextConverter.org$bdgenomics$adam$converters$VariantContextConverter$$convert$2(VariantContextConverter.scala:1824)
	at org.bdgenomics.adam.converters.VariantContextConverter$$anonfun$makeGenotypeFormatFn$1.apply(VariantContextConverter.scala:1877)
	at org.bdgenomics.adam.converters.VariantContextConverter$$anonfun$makeGenotypeFormatFn$1.apply(VariantContextConverter.scala:1877)
	at org.bdgenomics.adam.converters.VariantContextConverter$$anonfun$5.apply(VariantContextConverter.scala:368)
	at org.bdgenomics.adam.converters.VariantContextConverter$$anonfun$5.apply(VariantContextConverter.scala:367)
	at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
	at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
	at scala.collection.Iterator$class.foreach(Iterator.scala:893)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
	at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
	at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
	at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
	at scala.collection.AbstractTraversable.map(Traversable.scala:104)
	at org.bdgenomics.adam.converters.VariantContextConverter.convert(VariantContextConverter.scala:367)
	at org.bdgenomics.adam.rdd.ADAMContext$$anonfun$loadVcf$1$$anonfun$apply$15.apply(ADAMContext.scala:2073)
	at org.bdgenomics.adam.rdd.ADAMContext$$anonfun$loadVcf$1$$anonfun$apply$15.apply(ADAMContext.scala:2073)
	at scala.collection.Iterator$$anon$12.nextCur(Iterator.scala:434)
	at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:440)
	at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:439)
	at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1793)
	at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1158)
	at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1158)
	at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2069)
	at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2069)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
	at org.apache.spark.scheduler.Task.run(Task.scala:108)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:338)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

[heuermh]

Fixed by #1899