Throwing exception when matching input.sv.bam files are required but …

…missing. Prevents misconfigured pipelines from generating bad assembly results
PapenfussLab · Jun 5, 2018 · b8fba59 · b8fba59
1 parent 7e89071
commit b8fba59
Show file tree

Hide file tree

Showing 11 changed files with 50 additions and 5 deletions.
diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
 	<groupId>au.edu.wehi</groupId>
 	<artifactId>gridss</artifactId>
 	<packaging>jar</packaging>
-	<version>1.7.2-gridss-SNAPSHOT</version>
+	<version>1.7.2-gridss</version>
 	<name>gridss</name>
 	<url>https://github.com/PapenfussLab/gridss</url>
 	<properties>

diff --git a/src/main/java/au/edu/wehi/idsv/AssemblyEvidenceSource.java b/src/main/java/au/edu/wehi/idsv/AssemblyEvidenceSource.java
@@ -72,6 +72,7 @@ public AssemblyEvidenceSource(ProcessingContext processContext, List<SAMEvidence
 	 * @throws IOException 
 	 */
 	public void assembleBreakends(ExecutorService threadpool) throws IOException {
+		source.stream().forEach(ses -> ses.assertPreprocessingComplete());
 		if (threadpool == null) {
 			threadpool = MoreExecutors.newDirectExecutorService();
 		}

diff --git a/src/main/java/au/edu/wehi/idsv/FileSystemContext.java b/src/main/java/au/edu/wehi/idsv/FileSystemContext.java
@@ -99,6 +99,9 @@ public synchronized File getIntermediateDirectory(File file) {
 		return workingDir;
 	}
 	private static File getSource(File file) {
+		if (file == null) {
+			return null;
+		}
 		String source = file.getPath();
 		if (source.contains(INTERMEDIATE_DIR_SUFFIX)) {
 			source = source.substring(0, source.indexOf(INTERMEDIATE_DIR_SUFFIX));
@@ -107,6 +110,9 @@ private static File getSource(File file) {
 		return result;
 	}
 	private synchronized File getFile(String path) {
+		if (path == null) {
+			return null;
+		}
 		File file = new File(path);
 		file.getParentFile().mkdirs();
 		return file;

diff --git a/src/main/java/au/edu/wehi/idsv/SAMEvidenceSource.java b/src/main/java/au/edu/wehi/idsv/SAMEvidenceSource.java
@@ -284,10 +284,22 @@ public CloseableIterator<DirectedEvidence> iterator() {
 		return new AutoClosingIterator<>(eit, reader, it);
 	}
 	private SamReader getReader() {
-		File svFile = getContext().getFileSystemContext().getSVBam(getFile());
+		File svFile = getSVFile();
 		SamReader reader = getProcessContext().getSamReader(svFile.exists() ? svFile : getFile());
 		return reader;
 	}
+	public File getSVFile() {
+		if (getFile() == null) {
+			return null;
+		}
+		return getContext().getFileSystemContext().getSVBam(getFile());
+	}
+	public void assertPreprocessingComplete() {
+		File svFile = getSVFile();
+		if (svFile != null && !svFile.exists()) {
+			throw new IllegalStateException(String.format("Missing required file %s. See GRIDSS pipeline examples and documentation.", svFile));
+		}
+	}
 	private Iterator<DirectedEvidence> asEvidence(Iterator<SAMRecord> it) {
 		it = new BufferedIterator<>(it, 2); // TODO: remove when https://github.com/samtools/htsjdk/issues/760 is resolved 
 		it = Iterators.transform(it, r -> transform(r));

diff --git a/src/main/java/au/edu/wehi/idsv/VariantCaller.java b/src/main/java/au/edu/wehi/idsv/VariantCaller.java
@@ -35,6 +35,8 @@ public VariantCaller(ProcessingContext context, List<SAMEvidenceSource> samEvide
 		this.assemblyEvidence = assemblyEvidence;
 	}
 	public void callBreakends(File vcf, ExecutorService threadpool) throws IOException {
+		samEvidence.stream().forEach(ses -> ses.assertPreprocessingComplete());
+		assemblyEvidence.assertPreprocessingComplete();
 		log.info("Identifying Breakpoints");
 		if (threadpool == null) {
 			threadpool = MoreExecutors.newDirectExecutorService();

diff --git a/src/main/java/au/edu/wehi/idsv/util/FileHelper.java b/src/main/java/au/edu/wehi/idsv/util/FileHelper.java
@@ -56,12 +56,12 @@ private static void trymovesingle(File from, File to) throws IOException {
 	 * @param to
 	 * @throws IOException 
 	 */
-	public static void copy(File from, File to, boolean moveIndexes) throws IOException {
+	public static void copy(File from, File to, boolean copyIndexes) throws IOException {
 		if (!from.exists()) {
 			throw new IllegalArgumentException("Cannot copy nonexist file" + from.getAbsolutePath());
 		}
 		if (to.exists()) {
-			FileHelper.delete(to, moveIndexes);
+			FileHelper.delete(to, copyIndexes);
 		}
 		Files.copy(from, to);
 		copyIndex(from, to, ".bai");

diff --git a/src/main/java/gridss/AllocateEvidence.java b/src/main/java/gridss/AllocateEvidence.java
@@ -2,6 +2,7 @@
 
 import java.io.File;
 import java.util.Iterator;
+import java.util.List;
 import java.util.concurrent.ExecutorService;
 
 import org.broadinstitute.barclay.argparser.Argument;
@@ -54,7 +55,9 @@ public CloseableIterator<DirectedEvidence> getReadIterator() {
 		if (getContext().getVariantCallingParameters().callOnlyAssemblies) {
 			evidenceIt = SAMEvidenceSource.mergedIterator(ImmutableList.of(), false);
 		} else {
-			evidenceIt = SAMEvidenceSource.mergedIterator(ImmutableList.<SAMEvidenceSource>builder().addAll(getSamEvidenceSources()).build(), true);
+			List<SAMEvidenceSource> sources = getSamEvidenceSources();
+			sources.stream().forEach(ses -> ses.assertPreprocessingComplete());
+			evidenceIt = SAMEvidenceSource.mergedIterator(ImmutableList.<SAMEvidenceSource>builder().addAll(sources).build(), true);
 		}
 		if (Defaults.SANITY_CHECK_ITERATORS) {
 			evidenceIt = new AutoClosingIterator<>(

diff --git a/src/test/java/au/edu/wehi/idsv/AssemblyEvidenceSourceTest.java b/src/test/java/au/edu/wehi/idsv/AssemblyEvidenceSourceTest.java
@@ -21,6 +21,7 @@
 
 import au.edu.wehi.idsv.picard.InMemoryReferenceSequenceFile;
 import au.edu.wehi.idsv.sam.SAMRecordUtil;
+import au.edu.wehi.idsv.util.FileHelper;
 import htsjdk.samtools.SAMFileHeader;
 import htsjdk.samtools.SAMFileHeader.SortOrder;
 import htsjdk.samtools.SAMRecord;
@@ -38,6 +39,7 @@ public void setup() throws IOException {
 	public void should_write_breakend_bam() throws IOException {
 		createInput(RP(0, 1, 2, 1));
 		SAMEvidenceSource ses = new SAMEvidenceSource(getCommandlineContext(), input, null, 0);
+		FileHelper.copy(ses.getFile(), ses.getSVFile(), true);
 		AssemblyEvidenceSource aes = new AssemblyEvidenceSource(getCommandlineContext(), ImmutableList.of(ses), assemblyFile);
 		aes.assembleBreakends(null);
 		assertTrue(assemblyFile.exists());
@@ -46,6 +48,7 @@ public void should_write_breakend_bam() throws IOException {
 	public void breakend_bam_should_be_coordinate_sorted() throws IOException {
 		createInput(RP(0, 1, 2, 1));
 		SAMEvidenceSource ses = new SAMEvidenceSource(getCommandlineContext(), input, null, 0);
+		FileHelper.copy(ses.getFile(), ses.getSVFile(), true);
 		AssemblyEvidenceSource aes = new AssemblyEvidenceSource(getCommandlineContext(), ImmutableList.of(ses), assemblyFile);
 		aes.assembleBreakends(null);
 		try (SamReader r = SamReaderFactory.makeDefault().open(assemblyFile)) {
@@ -61,6 +64,7 @@ public void debruijn_should_generate_bam() throws IOException {
 		ProcessingContext pc = getCommandlineContext();
 		pc.getConfig().getAssembly().minReads = 1;
 		SAMEvidenceSource ses = new SAMEvidenceSource(pc, input, null, 0);
+		FileHelper.copy(ses.getFile(), ses.getSVFile(), true);
 		AssemblyEvidenceSource aes = new AssemblyEvidenceSource(pc, ImmutableList.of(ses), assemblyFile);
 		aes.assembleBreakends(null);
 
@@ -83,6 +87,7 @@ public void iterator_should_return_in_chr_order() throws IOException {
 		pc.getConfig().getAssembly().minReads = 1;
 		//pc.getRealignmentParameters().requireRealignment = false;
 		SAMEvidenceSource ses = new SAMEvidenceSource(pc, input, null, 0);
+		FileHelper.copy(ses.getFile(), ses.getSVFile(), true);
 		AssemblyEvidenceSource aes = new AssemblyEvidenceSource(pc, ImmutableList.of(ses), assemblyFile);
 		aes.assembleBreakends(null);
 		List<DirectedEvidence> list = Lists.newArrayList(aes.iterator());
@@ -99,6 +104,7 @@ public void should_not_write_filtered_assemblies() throws IOException {
 		ProcessingContext pc = getCommandlineContext();
 		pc.getAssemblyParameters().writeFiltered = false;
 		SAMEvidenceSource ses = new SAMEvidenceSource(pc, input, null, 0);
+		FileHelper.copy(ses.getFile(), ses.getSVFile(), true);
 		AssemblyEvidenceSource aes = new AssemblyEvidenceSource(pc, ImmutableList.of(ses), assemblyFile);
 		aes.assembleBreakends(null);
 		List<DirectedEvidence> contigs = Lists.newArrayList(aes.iterator());
@@ -229,6 +235,7 @@ public void should_adjust_assembly_contig_alignment_to_align_with_reference_homo
 				Read(0, 5, "5M7S")
 				);
 		SAMEvidenceSource ses = new SAMEvidenceSource(getCommandlineContext(), input, null, 0);
+		FileHelper.copy(ses.getFile(), ses.getSVFile(), true);
 		AssemblyEvidenceSource aes = new AssemblyEvidenceSource(getCommandlineContext(), ImmutableList.of(ses), assemblyFile);
 		aes.assembleBreakends(null);
 		List<SAMRecord> assemblies = getRecords(assemblyFile);
@@ -268,6 +275,7 @@ public void chunck_spanning_assemblies_should_not_be_repeated() throws IOExcepti
 		pc.getConfig().getAssembly().minReads = 1;
 		pc.getConfig().chunkSize = 100;
 		SAMEvidenceSource ses = new SAMEvidenceSource(pc, input, null, 0);
+		FileHelper.copy(ses.getFile(), ses.getSVFile(), true);
 		AssemblyEvidenceSource aes = new AssemblyEvidenceSource(pc, ImmutableList.of(ses), assemblyFile);
 		aes.assembleBreakends(null);
 		List<DirectedEvidence> list = Lists.newArrayList(aes.iterator());
@@ -284,6 +292,7 @@ public void parallel_assembly_should_not_affect_assembly_results() throws IOExce
 		pc.getConfig().getAssembly().minReads = 1;
 		pc.getConfig().chunkSize = 100;
 		SAMEvidenceSource ses = new SAMEvidenceSource(pc, input, null, 0);
+		FileHelper.copy(ses.getFile(), ses.getSVFile(), true);
 		AssemblyEvidenceSource aes = new AssemblyEvidenceSource(pc, ImmutableList.of(ses), assemblyFile);
 		ExecutorService threadpool = Executors.newFixedThreadPool(8);
 		aes.assembleBreakends(threadpool);
@@ -307,6 +316,7 @@ public void bounds_check_should_apply_to_final_assembly_SAMRecord() throws IOExc
 		pc.getConfig().getSoftClip().minAnchorIdentity = 0;
 		pc.getConfig().getSoftClip().minAverageQual = 0;
 		SAMEvidenceSource ses = new SAMEvidenceSource(pc, input, null, 0);
+		FileHelper.copy(ses.getFile(), ses.getSVFile(), true);
 		AssemblyEvidenceSource aes = new AssemblyEvidenceSource(pc, ImmutableList.of(ses), assemblyFile);
 		aes.assembleBreakends(null);
 		getRecords(assemblyFile);

diff --git a/src/test/java/au/edu/wehi/idsv/TestHelper.java b/src/test/java/au/edu/wehi/idsv/TestHelper.java
@@ -985,6 +985,9 @@ public CloseableIterator<DirectedEvidence> iterator(QueryInterval[] qi) {
 						QueryIntervalUtil.overlaps(qi, de.getBreakendSummary()))
 					);
 		}
+		@Override
+		public void assertPreprocessingComplete() {
+		}
 	}
 	public static class StubAssemblyEvidenceSource extends AssemblyEvidenceSource {
 		public int assemblyWindowSize = 10;

diff --git a/src/test/java/au/edu/wehi/idsv/visualisation/StaticDeBruijnPathGraphGexfExporterTest.java b/src/test/java/au/edu/wehi/idsv/visualisation/StaticDeBruijnPathGraphGexfExporterTest.java
@@ -18,6 +18,7 @@
 import au.edu.wehi.idsv.ProcessingContext;
 import au.edu.wehi.idsv.SAMEvidenceSource;
 import au.edu.wehi.idsv.sam.SAMFileUtil;
+import au.edu.wehi.idsv.util.FileHelper;
 import gridss.ComputeSamTags;
 import htsjdk.samtools.SAMFileHeader.SortOrder;
 import picard.sam.BuildBamIndex;
@@ -36,6 +37,7 @@ public void positional_should_export_dot() throws IOException {
 		});
 		ProcessingContext pc = getCommandlineContext();
 		SAMEvidenceSource ses = new SAMEvidenceSource(pc, input, null, 0);
+		FileHelper.copy(ses.getFile(), ses.getSVFile(), true);
 		AssemblyEvidenceSource aes = new AssemblyEvidenceSource(pc, ImmutableList.of(ses), output);
 		aes.assembleBreakends(null);
 		File dir = new File(super.testFolder.getRoot(), "visualisation");

diff --git a/src/test/java/gridss/AllocateEvidenceTest.java b/src/test/java/gridss/AllocateEvidenceTest.java
@@ -36,6 +36,7 @@
 import au.edu.wehi.idsv.VariantContextDirectedBreakpoint;
 import au.edu.wehi.idsv.VariantContextDirectedEvidence;
 import au.edu.wehi.idsv.util.AutoClosingIterator;
+import au.edu.wehi.idsv.util.FileHelper;
 import htsjdk.samtools.SAMFileHeader.SortOrder;
 import htsjdk.samtools.SAMRecord;
 
@@ -60,6 +61,7 @@ public void should_annotate_reads() throws IOException {
 				DP(0, 2, "5M5S", true, 1, 10, "5M", true));
 		SAMEvidenceSource ses = new SAMEvidenceSource(getContext(), input, null, 0);
 		ses.ensureMetrics();
+		FileHelper.copy(ses.getFile(), ses.getSVFile(), true);
 		File assemblyFile = new File(testFolder.getRoot(), "assembly.bam");
 		AssemblyEvidenceSource aes = new AssemblyEvidenceSource(pc, ImmutableList.of(ses), assemblyFile);
 		aes.assembleBreakends(null);
@@ -92,6 +94,7 @@ public void should_apply_filters() throws IOException {
 				DP(0, 1, "5M5S", true, 1, 10, "5M", true));
 		SAMEvidenceSource ses = new SAMEvidenceSource(getContext(), input, null, 0);
 		ses.ensureMetrics();
+		FileHelper.copy(ses.getFile(), ses.getSVFile(), true);
 		File assemblyFile = new File(testFolder.getRoot(), "assembly.bam");
 		AssemblyEvidenceSource aes = new AssemblyEvidenceSource(pc, ImmutableList.of(ses), assemblyFile);
 		aes.assembleBreakends(null);
@@ -184,6 +187,7 @@ public void should_filter_if_insufficient_reads() throws IOException {
 				DP(0, 1, "5M5S", true, 1, 10, "5M", true));
 		SAMEvidenceSource ses = new SAMEvidenceSource(getContext(), input, null, 0);
 		ses.ensureMetrics();
+		FileHelper.copy(ses.getFile(), ses.getSVFile(), true);
 		File assemblyFile = new File(testFolder.getRoot(), "assembly.bam");
 		AssemblyEvidenceSource aes = new AssemblyEvidenceSource(pc, ImmutableList.of(ses), assemblyFile);
 		aes.assembleBreakends(null);
@@ -215,6 +219,7 @@ public void insufficient_reads_filter_should_count_fragments() throws IOExceptio
 				withReadName("read", DP(0, 1, "6M4S", true, 1, 10, "5M", true)));
 		SAMEvidenceSource ses = new SAMEvidenceSource(getContext(), input, null, 0);
 		ses.ensureMetrics();
+		FileHelper.copy(ses.getFile(), ses.getSVFile(), true);
 		File assemblyFile = new File(testFolder.getRoot(), "assembly.bam");
 		AssemblyEvidenceSource aes = new AssemblyEvidenceSource(pc, ImmutableList.of(ses), assemblyFile);
 		aes.assembleBreakends(null);
@@ -243,6 +248,7 @@ public void should_filter_if_insufficient_quality() throws IOException {
 				DP(0, 1, "5M5S", true, 1, 10, "5M", true));
 		SAMEvidenceSource ses = new SAMEvidenceSource(getContext(), input, null, 0);
 		ses.ensureMetrics();
+		FileHelper.copy(ses.getFile(), ses.getSVFile(), true);
 		File assemblyFile = new File(testFolder.getRoot(), "assembly.bam");
 		AssemblyEvidenceSource aes = new AssemblyEvidenceSource(pc, ImmutableList.of(ses), assemblyFile);
 		aes.assembleBreakends(null);