Merge remote-tracking branch 'unstable/master'

build.xml

@@ -526,6 +526,7 @@
             <fileset dir="${lib.dir}">
                 <include name="scala-compiler-*.jar"/>
                 <include name="scala-library-*.jar"/>
+                <include name="scala-reflect-*.jar"/>
             </fileset>
         </path>
         <taskdef resource="scala/tools/ant/antlib.xml">
@@ -537,7 +538,7 @@
     <target name="scala.compile" depends="init,resolve,gatk.compile,queue-extensions.generate,init.scala.compile" if="include.scala">
         <mkdir dir="${scala.classes}"/>
         <echo>Building Scala...</echo>
-        <scalac fork="true" jvmargs="-Xmx512m" destdir="${scala.classes}" classpathref="scala.dependencies" deprecation="yes" unchecked="yes">
+        <scalac fork="true" jvmargs="-Xmx512m" destdir="${scala.classes}" classpathref="scala.dependencies" deprecation="yes" unchecked="yes" addparams="-feature">
             <src refid="scala.source.path" />
             <src path="${queue-extensions.source.dir}" />
             <include name="**/*.scala" />
@@ -1218,7 +1219,7 @@
 
     <target name="test.scala.compile" depends="test.java.compile,scala.compile" if="include.scala">
         <echo message="Scala: Compiling test cases!"/>
-        <scalac fork="true" jvmargs="-Xmx512m" destdir="${scala.test.classes}" deprecation="yes" unchecked="yes">
+        <scalac fork="true" jvmargs="-Xmx512m" destdir="${scala.test.classes}" deprecation="yes" unchecked="yes" addparams="-feature">
 	        <src refid="scala.test.source.path" />
             <classpath>
                 <path refid="build.results"/>
@@ -1414,6 +1415,13 @@
         <run-test testtype="${pipetype}" outputdir="${report}/${pipetype}" runfailed="false"/>
     </target>
 
+    <target name="knowledgebasetest" depends="test.compile,test.init" description="Run knowledge base tests">
+        <condition property="ktype" value="*KnowledgeBaseTest" else="${single}">
+            <not><isset property="single"/></not>
+        </condition>
+        <run-test testtype="${ktype}" outputdir="${report}/${ktype}" runfailed="false"/>
+    </target>
+
     <target name="failed-unit" depends="test.compile,test.init">
         <run-test testtype="${report}/*UnitTest/testng-failed.xml" outputdir="${report}/failed_rerun" runfailed="true"/>
     </target>

ivy.xml

@@ -82,8 +82,8 @@
         <dependency org="net.sf.gridscheduler" name="drmaa" rev="latest.integration"/>
 
         <!-- Scala dependancies -->
-        <dependency org="org.scala-lang" name="scala-compiler" rev="2.9.2"/>
-        <dependency org="org.scala-lang" name="scala-library" rev="2.9.2"/>
+        <dependency org="org.scala-lang" name="scala-compiler" rev="2.10.2"/>
+        <dependency org="org.scala-lang" name="scala-library" rev="2.10.2"/>
 
         <!-- testing and evaluation dependencies -->
         <dependency org="org.testng" name="testng" rev="6.8"/>

public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java

@@ -35,6 +35,8 @@
 import org.broadinstitute.sting.utils.QualityUtils;
 import org.broadinstitute.sting.utils.baq.BAQ;
 import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.variant.GATKVCFIndexType;
+import org.broadinstitute.sting.utils.variant.GATKVCFUtils;
 
 import java.io.File;
 import java.util.ArrayList;
@@ -119,21 +121,28 @@ public GATKArgumentCollection() {
     // Downsampling Arguments
     //
     // --------------------------------------------------------------------------------------------------------------
-    @Argument(fullName = "downsampling_type", shortName="dt", doc="Type of reads downsampling to employ at a given locus.  Reads will be selected randomly to be removed from the pile based on the method described here", required = false)
+    /**
+     * Reads will be selected randomly to be removed from the pile based on the method described here.
+     */
+    @Argument(fullName = "downsampling_type", shortName="dt", doc="Type of reads downsampling to employ at a given locus", required = false)
     public DownsampleType downsamplingType = null;
 
     @Argument(fullName = "downsample_to_fraction", shortName = "dfrac", doc = "Fraction [0.0-1.0] of reads to downsample to", required = false)
     public Double downsampleFraction = null;
 
+    /**
+     * For locus-based traversals (LocusWalkers and ActiveRegionWalkers), downsample_to_coverage controls the
+     * maximum depth of coverage at each locus. For read-based traversals (ReadWalkers), it controls the
+     * maximum number of reads sharing the same alignment start position. For ReadWalkers you will typically need to use
+     * much lower dcov values than you would with LocusWalkers to see an effect. Note that this downsampling option does
+     * not produce an unbiased random sampling from all available reads at each locus: instead, the primary goal of the
+     * to-coverage downsampler is to maintain an even representation of reads from all alignment start positions when
+     * removing excess coverage. For a truly unbiased random sampling of reads, use -dfrac instead. Also note
+     * that the coverage target is an approximate goal that is not guaranteed to be met exactly: the downsampling
+     * algorithm will under some circumstances retain slightly more or less coverage than requested.
+     */
     @Argument(fullName = "downsample_to_coverage", shortName = "dcov",
-              doc = "Coverage [integer] to downsample to. For locus-based traversals (eg., LocusWalkers and ActiveRegionWalkers)," +
-                    "this controls the maximum depth of coverage at each locus. For non-locus-based traversals (eg., ReadWalkers), " +
-                    "this controls the maximum number of reads sharing the same alignment start position. Note that this downsampling " +
-                    "option does NOT produce an unbiased random sampling from all available reads at each locus: instead, the primary goal of " +
-                    "the to-coverage downsampler is to maintain an even representation of reads from all alignment start positions " +
-                    "when removing excess coverage. For a true across-the-board unbiased random sampling of reads, use -dfrac instead. " +
-                    "Also note that the coverage target is an approximate goal that is not guaranteed to be met exactly: the downsampling " +
-                    "algorithm will under some circumstances retain slightly more coverage than requested.",
+              doc = "Coverage [integer] to downsample to per locus (for locus walkers) or per alignment start position (for read walkers)",
               required = false)
     public Integer downsampleCoverage = null;
 
@@ -447,5 +456,28 @@ public void setDownsamplingMethod(DownsamplingMethod method) {
     @Hidden
     public boolean generateShadowBCF = false;
     // TODO -- remove all code tagged with TODO -- remove me when argument generateShadowBCF is removed
+
+    // --------------------------------------------------------------------------------------------------------------
+    //
+    // VCF/BCF index parameters
+    //
+    // --------------------------------------------------------------------------------------------------------------
+
+    /**
+     * Specify the Tribble indexing strategy to use for VCFs.
+     *
+     * LINEAR creates a LinearIndex with bins of equal width, specified by the Bin Width parameter
+     * INTERVAL creates an IntervalTreeIndex with bins with an equal amount of features, specified by the Features Per Bin parameter
+     * DYNAMIC_SEEK attempts to optimize for minimal seek time by choosing an appropriate strategy and parameter (user-supplied parameter is ignored)
+     * DYNAMIC_SIZE attempts to optimize for minimal index size by choosing an appropriate strategy and parameter (user-supplied parameter is ignored)
+     */
+
+    @Argument(fullName="variant_index_type",shortName = "variant_index_type",doc="which type of IndexCreator to use for VCF/BCF indices",required=false)
+    @Advanced
+    public GATKVCFIndexType variant_index_type = GATKVCFUtils.DEFAULT_INDEX_TYPE;
+
+    @Argument(fullName="variant_index_parameter",shortName = "variant_index_parameter",doc="the parameter (bin width or features per bin) to pass to the VCF/BCF IndexCreator",required=false)
+    @Advanced
+    public int variant_index_parameter = GATKVCFUtils.DEFAULT_INDEX_PARAMETER;
 }
 

public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BAMScheduler.java

@@ -246,6 +246,14 @@ else if(locusIterator.hasNext())
      */
     private PeekableIterator<BAMScheduleEntry> bamScheduleIterator = null;
 
+    /**
+     * Clean up underlying BAMSchedule file handles.
+     */
+    public void close() {
+        if(bamScheduleIterator != null)
+            bamScheduleIterator.close();
+    }
+
     /**
      * Get the next overlapping tree of bins associated with the given BAM file.
      * @param currentLocus The actual locus for which to check overlap.

public/java/src/org/broadinstitute/sting/gatk/datasources/reads/IntervalSharder.java

@@ -62,6 +62,9 @@ private IntervalSharder(final BAMScheduler scheduler, final GenomeLocParser pars
         wrappedIterator = new PeekableIterator<FilePointer>(scheduler);
         this.parser = parser;
     }
+    public void close() {
+      wrappedIterator.close();
+    }
 
     public boolean hasNext() {
         return wrappedIterator.hasNext();

public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java

@@ -352,6 +352,14 @@ public SAMDataSource(
         resourcePool.releaseReaders(readers);
     }
 
+    public void close() {
+        SAMReaders readers = resourcePool.getAvailableReaders();
+        for(SAMReaderID readerID: readerIDs) {
+            SAMFileReader reader = readers.getReader(readerID);
+            reader.close();
+        }
+    }
+
     /**
      * Returns Reads data structure containing information about the reads data sources placed in this pool as well as
      * information about how they are downsampled, sorted, and filtered

public/java/src/org/broadinstitute/sting/gatk/datasources/reads/ShardBalancer.java

@@ -43,4 +43,7 @@ public void initialize(final SAMDataSource readsDataSource, final Iterator<FileP
         this.filePointers = new PeekableIterator<FilePointer>(filePointers);
         this.parser = parser;
     }
+    public void close() {
+      this.filePointers.close();
+    }
 }

public/java/src/org/broadinstitute/sting/gatk/io/storage/StorageFactory.java

@@ -62,7 +62,7 @@ public static <T> Storage<T> createStorage( Stub<T> stub ) {
      * @param <T> Type of the stream to create.
      * @return Storage object with a facade of type T.
      */
-        public static <T> Storage<T> createStorage( Stub<T> stub, File file ) {
+     public static <T> Storage<T> createStorage( Stub<T> stub, File file ) {
         Storage storage;
 
         if(stub instanceof OutputStreamStub) {

public/java/src/org/broadinstitute/sting/gatk/io/storage/VariantContextWriterStorage.java

@@ -133,14 +133,21 @@ private VariantContextWriter vcfWriterToFile(final VariantContextWriterStub stub
 
         // The GATK/Tribble can't currently index block-compressed files on the fly.  Disable OTF indexing even if the user explicitly asked for it.
         EnumSet<Options> options = stub.getWriterOptions(indexOnTheFly);
-        VariantContextWriter writer = VariantContextWriterFactory.create(file, this.stream, stub.getMasterSequenceDictionary(), options);
+        VariantContextWriter writer = VariantContextWriterFactory.create(file, this.stream, stub.getMasterSequenceDictionary(), stub.getIndexCreator(), options);
 
         // if the stub says to test BCF, create a secondary writer to BCF and an 2 way out writer to send to both
         // TODO -- remove me when argument generateShadowBCF is removed
         if ( stub.alsoWriteBCFForTest() && ! VariantContextWriterFactory.isBCFOutput(file, options)) {
             final File bcfFile = BCF2Utils.shadowBCF(file);
             if ( bcfFile != null ) {
-                VariantContextWriter bcfWriter = VariantContextWriterFactory.create(bcfFile, stub.getMasterSequenceDictionary(), options);
+                FileOutputStream bcfStream;
+                try {
+                    bcfStream = new FileOutputStream(bcfFile);
+                } catch (FileNotFoundException e) {
+                    throw new RuntimeException(bcfFile + ": Unable to create BCF writer", e);
+                }
+
+                VariantContextWriter bcfWriter = VariantContextWriterFactory.create(bcfFile, bcfStream, stub.getMasterSequenceDictionary(), stub.getIndexCreator(), options);
                 writer = new TestWriter(writer, bcfWriter);
             }
         }

public/java/src/org/broadinstitute/sting/gatk/io/stubs/VariantContextWriterStub.java

@@ -26,6 +26,7 @@
 package org.broadinstitute.sting.gatk.io.stubs;
 
 import net.sf.samtools.SAMSequenceDictionary;
+import org.broad.tribble.index.IndexCreator;
 import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
 import org.broadinstitute.sting.gatk.io.OutputTracker;
 import org.broadinstitute.sting.utils.variant.GATKVCFUtils;
@@ -70,6 +71,17 @@ public class VariantContextWriterStub implements Stub<VariantContextWriter>, Var
      */
     private final PrintStream genotypeStream;
 
+    /**
+     * A hack: push the argument sources into the VCF header so that the VCF header
+     * can rebuild the command-line arguments.
+     */
+    private final Collection<Object> argumentSources;
+
+    /**
+     * Which IndexCreator to use
+     */
+    private final IndexCreator indexCreator;
+
     /**
      * The cached VCF header (initialized to null)
      */
@@ -80,12 +92,6 @@ public class VariantContextWriterStub implements Stub<VariantContextWriter>, Var
      */
     private boolean isCompressed = false;
 
-    /**
-     * A hack: push the argument sources into the VCF header so that the VCF header
-     * can rebuild the command-line arguments.
-     */
-    private final Collection<Object> argumentSources;
-
     /**
      * Should the header be written out?  A hidden argument.
      */
@@ -118,6 +124,7 @@ public VariantContextWriterStub(GenomeAnalysisEngine engine, File genotypeFile,
         this.engine = engine;
         this.genotypeFile = genotypeFile;
         this.genotypeStream = null;
+        this.indexCreator = GATKVCFUtils.getIndexCreator(engine.getArguments().variant_index_type, engine.getArguments().variant_index_parameter, genotypeFile);
         this.argumentSources = argumentSources;
     }
 
@@ -132,6 +139,7 @@ public VariantContextWriterStub(GenomeAnalysisEngine engine, OutputStream genoty
         this.engine = engine;
         this.genotypeFile = null;
         this.genotypeStream = new PrintStream(genotypeStream);
+        this.indexCreator = null;
         this.argumentSources = argumentSources;
     }
 
@@ -175,6 +183,10 @@ public void setForceBCF(boolean forceBCF) {
         this.forceBCF = forceBCF;
     }
 
+    public IndexCreator getIndexCreator() {
+        return indexCreator;
+    }
+
     /**
      * Gets the master sequence dictionary from the engine associated with this stub
      * @link GenomeAnalysisEngine.getMasterSequenceDictionary

public/java/src/org/broadinstitute/sting/gatk/report/GATKReportGatherer.java

@@ -42,23 +42,21 @@ public void gather(List<File> inputs, File output) {
         try {
             o = new PrintStream(output);
         } catch (FileNotFoundException e) {
-            throw new UserException("File to be output by CoverageByRG Gather function was not found");
+            throw new UserException(String.format("File %s to be output by GATKReportGatherer function was not found", output));
         }
 
         GATKReport current = new GATKReport();
         boolean isFirst = true;
         for (File input : inputs) {
-
-            // If the table is empty
             if (isFirst) {
                 current = new GATKReport(input);
                 isFirst = false;
             } else {
-                GATKReport toAdd = new GATKReport(input);
-                current.concat(toAdd);
+                current.concat(new GATKReport(input));
             }
         }
 
         current.print(o);
+        o.close();
     }
 }

public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java

@@ -86,7 +86,7 @@ public final class TraverseActiveRegions<M, T> extends TraversalEngine<M,T,Activ
     private int maxRegionSize = -1;
     private int minRegionSize = -1;
 
-    private final LinkedList<ActiveRegion> workQueue = new LinkedList<ActiveRegion>();
+    private final LinkedList<ActiveRegion> workQueue = new LinkedList<>();
 
     private TAROrderedReadCache myReads = null;
 

public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java

@@ -180,4 +180,6 @@ public final GenomeLocSortedSet extendIntervals( final GenomeLocSortedSet interv
         }
         return IntervalUtils.sortAndMergeIntervals(genomeLocParser, allIntervals, IntervalMergingRule.ALL);
     }
+
+
 }