Evoquer: a BigQuery-based joint calling tool

.travis.yml

@@ -21,6 +21,7 @@ env:
   - SCALA_VERSION=2.11 RUN_CNN_WDL=true TESTS_REQUIRE_GCLOUD=true
   global:
   #gradle needs this
+  - NO_GCE_CHECK=true
   - TERM=dumb
   #limit gradle jvm memory and disable daemon
   - GRADLE_OPTS="-Xmx2048m -Dorg.gradle.daemon=false"

build.gradle

@@ -60,16 +60,19 @@ repositories {
 final htsjdkVersion = System.getProperty('htsjdk.version','2.20.3')
 final picardVersion = System.getProperty('picard.version','2.21.1')
 final barclayVersion = System.getProperty('barclay.version','2.1.0')
-final sparkVersion = System.getProperty('spark.version', '2.4.3')
+final sparkVersion = System.getProperty('spark.version', '2.4.4')
 final scalaVersion = System.getProperty('scala.version', '2.11')
-final hadoopVersion = System.getProperty('hadoop.version', '2.8.2')
+final hadoopVersion = System.getProperty('hadoop.version', '3.2.1')
 final disqVersion = System.getProperty('disq.version','0.3.3')
 final genomicsdbVersion = System.getProperty('genomicsdb.version','1.1.2.2')
+final bigQueryVersion = System.getProperty('bigQuery.version', '1.69.0')
+final guavaVersion = System.getProperty('guava.version', '27.1-jre')
 final testNGVersion = '6.11'
+
 // Using the shaded version to avoid conflicts between its protobuf dependency
 // and that of Hadoop/Spark (either the one we reference explicitly, or the one
 // provided by dataproc).
-final googleCloudNioDependency = 'com.google.cloud:google-cloud-nio:0.107.0-alpha:shaded'
+final googleCloudNioDependency = 'com.google.cloud:google-cloud-nio:0.117.0-alpha:shaded'
 
 final baseJarName = 'gatk'
 final secondaryBaseJarName = 'hellbender'
@@ -163,11 +166,10 @@ ensureBuildPrerequisites(largeResourcesFolder, buildPrerequisitesMessage)
 configurations.all {
     resolutionStrategy {
         // the snapshot folder contains a dev version of guava, we don't want to use that.
-        force 'com.google.guava:guava:18.0'
+        force 'com.google.guava:guava:' + guavaVersion
         // force the htsjdk version so we don't get a different one transitively
         force 'com.github.samtools:htsjdk:' + htsjdkVersion
-        // later versions explode Hadoop
-        force 'com.google.protobuf:protobuf-java:3.0.0-beta-1'
+        force 'com.google.protobuf:protobuf-java:3.8.0'
         // force testng dependency so we don't pick up a different version via GenomicsDB
         force 'org.testng:testng:' + testNGVersion
         force 'org.broadinstitute:barclay:' + barclayVersion
@@ -250,18 +252,21 @@ dependencies {
         exclude module: 'protobuf-java'
     }
     compile 'com.opencsv:opencsv:3.4'
-    compile 'com.google.guava:guava:18.0'
+    compile 'com.google.guava:guava:' + guavaVersion
     compile 'com.github.samtools:htsjdk:'+ htsjdkVersion
     compile(googleCloudNioDependency) {
         transitive = false
     }
 
+    compile 'com.google.cloud:google-cloud-bigquery:' + bigQueryVersion
+    compile 'com.google.cloud:google-cloud-bigquerystorage:0.92.0-beta'
+
     compile "gov.nist.math.jama:gov.nist.math.jama:1.1.1"
 
     // this comes built-in when running on Google Dataproc, but the library
     // allows us to read from GCS also when testing locally (or on non-Dataproc clusters,
     // should we want to)
-    compile 'com.google.cloud.bigdataoss:gcs-connector:1.6.3-hadoop2'
+    compile 'com.google.cloud.bigdataoss:gcs-connector:1.9.4-hadoop3'
 
     compile 'org.apache.logging.log4j:log4j-api:2.3'
     compile 'org.apache.logging.log4j:log4j-core:2.3'
@@ -351,6 +356,7 @@ dependencies {
 
     // natural sort
     compile('net.grey-panther:natural-comparator:1.1')
+    compile('com.fasterxml.jackson.module:jackson-module-scala_' + scalaVersion + ':2.9.8')
 
     testUtilsCompile sourceSets.main.output
     testUtilsCompile 'org.testng:testng:' + testNGVersion

scripts/unsupported/evoquer/IntervalSharder.java

@@ -0,0 +1,26 @@
+public class IntervalSharder {
+    public static void main(String[] args) {
+        if ( args.length != 3 ) {
+            System.out.println("Usage: IntervalSharder contig contigLength shardSize");
+            System.exit(1);
+        }
+
+        final String contig = args[0];
+        final int contigLength = Integer.parseInt(args[1]);
+        final int shardSize = Integer.parseInt(args[2]);
+
+        int currentStart = 1 - shardSize;
+        int currentEnd = 0;
+
+        while ( currentEnd < contigLength ) {
+            currentStart += shardSize;
+            currentEnd += shardSize;
+
+            if ( currentEnd > contigLength ) {
+                currentEnd = contigLength;
+            }
+
+            System.out.printf("%s:%d-%d\n", contig, currentStart, currentEnd);
+        } 
+    }
+}