Skip to content

Commit

Permalink
Json report (#284)
Browse files Browse the repository at this point in the history
This fixes #275
This fixes #268
  • Loading branch information
dbolotin authored and PoslavskySV committed Nov 24, 2017
1 parent c313da1 commit a296012
Show file tree
Hide file tree
Showing 21 changed files with 736 additions and 148 deletions.
5 changes: 4 additions & 1 deletion CHANGELOG_CURRENT
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
Added chains column in export (-chains and -topChains)
Report file appends made atomic. Now you can safely specify the same report file in several invocations of MiXCR even if you run several instances of MiXCR in parallel. Should also work for NFS etc if file locking is supported.
(beta) Added JSON-formatted reports. Use `--json-report` option in `align`, `extendAlignments`, `assemblePartial` or `assemble` actions to append json report to the end of the file ([jsonl](http://jsonlines.org/)). This feature intended ot simplify batch analysis, and analysis automation. E.g. [jq](https://stedolan.github.io/jq/) util can be used to aggregate information from multi-json-report file.
Added chains column in export (`-chains` and `-topChains`)
minor: Several new counters added to `align` report
minor: Added proper filtering for alleles with wildcard symbols in `featureToAlign` region
minor: Fixed bug in align report (wrong total count in rare cases) #265
minor: Documentation added for `filterAlignments`
120 changes: 120 additions & 0 deletions src/main/java/com/milaboratory/mixcr/cli/AbstractActionReport.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
/*
* Copyright (c) 2014-2017, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
* (here and after addressed as Inventors)
* All Rights Reserved
*
* Permission to use, copy, modify and distribute any part of this program for
* educational, research and non-profit purposes, by non-profit institutions
* only, without fee, and without a written agreement is hereby granted,
* provided that the above copyright notice, this paragraph and the following
* three paragraphs appear in all copies.
*
* Those desiring to incorporate this work into commercial products or use for
* commercial purposes should contact the Inventors using one of the following
* email addresses: chudakovdm@mail.ru, chudakovdm@gmail.com
*
* IN NO EVENT SHALL THE INVENTORS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT,
* SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS,
* ARISING OUT OF THE USE OF THIS SOFTWARE, EVEN IF THE INVENTORS HAS BEEN
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* THE SOFTWARE PROVIDED HEREIN IS ON AN "AS IS" BASIS, AND THE INVENTORS HAS
* NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR
* MODIFICATIONS. THE INVENTORS MAKES NO REPRESENTATIONS AND EXTENDS NO
* WARRANTIES OF ANY KIND, EITHER IMPLIED OR EXPRESS, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A
* PARTICULAR PURPOSE, OR THAT THE USE OF THE SOFTWARE WILL NOT INFRINGE ANY
* PATENT, TRADEMARK OR OTHER RIGHTS.
*/
package com.milaboratory.mixcr.cli;

import com.fasterxml.jackson.annotation.JsonFormat;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.milaboratory.mixcr.util.MiXCRVersionInfo;
import com.milaboratory.util.TimeUtils;

import java.util.Date;

public abstract class AbstractActionReport implements ActionReport {
private Date date = new Date();
private String commandLine;
private String[] inputFiles, outputFiles;
private long startMillis, finishMillis;

@JsonProperty("timestamp")
@JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss.SSSZ")
public Date getDate() {
return date;
}

@JsonProperty("commandLine")
public String getCommandLine() {
return commandLine;
}

@JsonProperty("inputFiles")
public String[] getInputFiles() {
return inputFiles;
}

@JsonProperty("outputFiles")
public String[] getOutputFiles() {
return outputFiles;
}

@JsonProperty("executionTimeMillis")
public long getExecutionTimeMillis() {
return finishMillis - startMillis;
}

@JsonProperty("version")
public String getVersion() {
return MiXCRVersionInfo.get().getShortestVersionString();
}

public void setCommandLine(String commandLine) {
this.commandLine = commandLine;
}

public void setInputFiles(String... inputFiles) {
this.inputFiles = inputFiles;
}

public void setOutputFiles(String... outputFiles) {
this.outputFiles = outputFiles;
}

public void setStartMillis(long startMillis) {
this.startMillis = startMillis;
}

public void setFinishMillis(long finishMillis) {
this.finishMillis = finishMillis;
}

private static String join(String[] strs) {
StringBuilder builder = new StringBuilder();
for (int i = 0; ; ++i) {
builder.append(strs[i]);
if (i == strs.length - 1)
break;
builder.append(',');
}
return builder.toString();
}

public void writeSuperReport(ReportHelper helper) {
if (!helper.isStdout())
helper.writeField("Analysis Date", getDate())
.writeField("Input file(s)", join(getInputFiles()))
.writeField("Output file(s)", join(getOutputFiles()))
.writeField("Version", getVersion());

if (!helper.isStdout())
if (getCommandLine() != null)
helper.writeField("Command line arguments", getCommandLine());

if (getExecutionTimeMillis() >= 0)
helper.writeField("Analysis time", TimeUtils.nanoTimeToString(getExecutionTimeMillis() * 1000_000));
}
}
43 changes: 26 additions & 17 deletions src/main/java/com/milaboratory/mixcr/cli/ActionAlign.java
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,14 @@
import com.milaboratory.mixcr.vdjaligners.VDJCAlignmentResult;
import com.milaboratory.mixcr.vdjaligners.VDJCParametersPresets;
import com.milaboratory.util.CanReportProgress;
import com.milaboratory.util.GlobalObjectMappers;
import com.milaboratory.util.SmartProgressReporter;
import io.repseq.core.*;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.*;

import static cc.redberry.pipe.CUtils.chunked;
Expand Down Expand Up @@ -177,9 +181,13 @@ else if (featureSequence.containsWildcards())
"(turn on verbose warnings by adding --verbose option).");

AlignerReport report = new AlignerReport();
aligner.setEventsListener(report);
report.setStartMillis(beginTimestamp);
report.setInputFiles(actionParameters.getInputsForReport());
report.setOutputFiles(actionParameters.getOutputsForReport());
report.setCommandLine(helper.getCommandLineArguments());

ChainUsageStats chainsStatistics = new ChainUsageStats();
// Attaching report to aligner
aligner.setEventsListener(report);

try (SequenceReaderCloseable<? extends SequenceRead> reader = actionParameters.createReader();

Expand Down Expand Up @@ -231,8 +239,6 @@ public long getIndex(VDJCAlignmentResult o) {
}
}

chainsStatistics.put(alignment);

if (alignment.isChimera())
report.onChimera();

Expand All @@ -249,15 +255,17 @@ public long getIndex(VDJCAlignmentResult o) {
writer.setNumberOfProcessedReads(reader.getNumberOfReads());
}

long time = System.currentTimeMillis() - beginTimestamp;
report.setFinishMillis(System.currentTimeMillis());

// Writing report to stout
System.out.println("============= Report ==============");
Util.writeReportToStdout(time, report, chainsStatistics);
Util.writeReportToStdout(report);

if (actionParameters.report != null)
Util.writeReport(actionParameters.getInputForReport(), actionParameters.getOutputName(),
helper.getCommandLineArguments(), actionParameters.report, time, report, chainsStatistics);
Util.writeReport(actionParameters.report, report);

if (actionParameters.jsonReport != null)
Util.writeJsonReport(actionParameters.jsonReport, report);
}

public static String[] extractDescriptions(SequenceRead r) {
Expand Down Expand Up @@ -317,6 +325,10 @@ public static class AlignParameters extends ActionParametersWithOutput {
names = {"-r", "--report"})
public String report;

@Parameter(description = "JSON report file.",
names = {"--json-report"})
public String jsonReport = null;

@Parameter(description = "Species (organism), as specified in library file or taxon id. " +
"Possible values: hs, HomoSapiens, musmusculus, mmu, hsa, 9606, 10090 etc..",
names = {"-s", "--species"})
Expand Down Expand Up @@ -376,15 +388,12 @@ public VDJCAlignerParameters getAlignerParameters() {
return params;
}

public String getInputForReport() {
StringBuilder builder = new StringBuilder();
for (int i = 0; ; ++i) {
builder.append(parameters.get(i));
if (i == parameters.size() - 2)
break;
builder.append(',');
}
return builder.toString();
public String[] getInputsForReport() {
return parameters.subList(0, parameters.size() - 1).toArray(new String[parameters.size() - 1]);
}

public String[] getOutputsForReport() {
return new String[]{getOutputName()};
}

public Boolean getNoMerge() {
Expand Down
35 changes: 23 additions & 12 deletions src/main/java/com/milaboratory/mixcr/cli/ActionAssemble.java
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ public void go(ActionHelper helper) throws Exception {
// Extracting V/D/J/C gene list from input vdjca file
final List<VDJCGene> genes;
final VDJCAlignerParameters alignerParameters;
try(VDJCAlignmentsReader reader = new VDJCAlignmentsReader(actionParameters.getInputFileName(),
try (VDJCAlignmentsReader reader = new VDJCAlignmentsReader(actionParameters.getInputFileName(),
VDJCLibraryRegistry.getDefault())) {
genes = reader.getUsedGenes();
// Saving aligner parameters to correct assembler parameters
Expand All @@ -91,10 +91,15 @@ public void go(ActionHelper helper) throws Exception {
}

// Performing assembly
try(CloneAssembler assembler = new CloneAssembler(assemblerParameters,
try (CloneAssembler assembler = new CloneAssembler(assemblerParameters,
actionParameters.readsToClonesMapping != null, genes, alignerParameters.getFeaturesToAlignMap())) {
// Creating event listener to collect run statistics
CloneAssemblerReport report = new CloneAssemblerReport();
report.setStartMillis(beginTimestamp);
report.setInputFiles(new String[]{actionParameters.getInputFileName()});
report.setOutputFiles(new String[]{actionParameters.getOutputFileName()});
report.setCommandLine(helper.getCommandLineArguments());

assembler.setListener(report);

// Running assembler
Expand All @@ -107,34 +112,36 @@ public void go(ActionHelper helper) throws Exception {
// Getting results
final CloneSet cloneSet = assemblerRunner.getCloneSet();

ChainUsageStats chainsStatistics = new ChainUsageStats();
for (Clone clone : cloneSet)
chainsStatistics.put(clone);
// Passing final cloneset to assemble last pieces of statistics for report
report.onClonesetFinished(cloneSet);

// Writing results
try(CloneSetIO.CloneSetWriter writer = new CloneSetIO.CloneSetWriter(cloneSet, actionParameters.getOutputFileName())) {
try (CloneSetIO.CloneSetWriter writer = new CloneSetIO.CloneSetWriter(cloneSet, actionParameters.getOutputFileName())) {
SmartProgressReporter.startProgressReport(writer);
writer.write();
}

// Writing report
long time = System.currentTimeMillis() - beginTimestamp;

report.setFinishMillis(System.currentTimeMillis());

assert cloneSet.getClones().size() == report.getCloneCount();

report.setTotalReads(alignmentsProvider.getTotalNumberOfReads());

// Writing report to stout
System.out.println("============= Report ==============");
Util.writeReportToStdout(time, report, chainsStatistics);
Util.writeReportToStdout(report);

if (actionParameters.report != null)
Util.writeReport(actionParameters.getInputFileName(), actionParameters.getOutputFileName(),
helper.getCommandLineArguments(), actionParameters.report, time, report, chainsStatistics);
Util.writeReport(actionParameters.report, report);

if (actionParameters.jsonReport != null)
Util.writeJsonReport(actionParameters.jsonReport, report);

// Writing raw events (not documented feature)
if (actionParameters.events != null)
try(PipeWriter<ReadToCloneMapping> writer = new PipeWriter<>(actionParameters.events)) {
try (PipeWriter<ReadToCloneMapping> writer = new PipeWriter<>(actionParameters.events)) {
CUtils.drain(assembler.getAssembledReadsPort(), writer);
}

Expand Down Expand Up @@ -168,10 +175,14 @@ public static final class AssembleParameters extends ActionParametersWithOutput
names = {"-t", "--threads"}, validateWith = PositiveInteger.class)
public int threads = Runtime.getRuntime().availableProcessors();

@Parameter(description = "Report file.",
@Parameter(description = "Report file",
names = {"-r", "--report"})
public String report;

@Parameter(description = "JSON report file.",
names = {"--json-report"})
public String jsonReport = null;

@Parameter(description = ".",
names = {"-e", "--events"}, hidden = true)
public String events;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,15 @@ public void go(ActionHelper helper) throws Exception {
}
}

long start = System.currentTimeMillis();
try (PartialAlignmentsAssembler assembler = new PartialAlignmentsAssembler(assemblerParameters,
parameters.getOutputFileName(), !parameters.doDropPartial(),
parameters.getOverlappedOnly())) {
ReportWrapper report = new ReportWrapper(command(), assembler);
report.setStartMillis(beginTimestamp);
report.setInputFiles(parameters.getInputFileName());
report.setOutputFiles(parameters.getOutputFileName());
report.setCommandLine(helper.getCommandLineArguments());

try (VDJCAlignmentsReader reader = new VDJCAlignmentsReader(parameters.getInputFileName())) {
SmartProgressReporter.startProgressReport("Building index", reader);
assembler.buildLeftPartsIndex(reader);
Expand All @@ -57,17 +62,17 @@ public void go(ActionHelper helper) throws Exception {
assembler.searchOverlaps(reader);
}

if (parameters.report != null)
Util.writeReport(parameters.getInputFileName(), parameters.getOutputFileName(),
helper.getCommandLineArguments(), parameters.report,
System.currentTimeMillis() - start, assembler
);

long time = System.currentTimeMillis() - beginTimestamp;
report.setFinishMillis(System.currentTimeMillis());

// Writing report to stout
System.out.println("============= Report ==============");
Util.writeReportToStdout(time, assembler);
Util.writeReportToStdout(report);

if (parameters.report != null)
Util.writeReport(parameters.report, report);

if (parameters.jsonReport != null)
Util.writeJsonReport(parameters.jsonReport, report);
}
}

Expand All @@ -93,6 +98,10 @@ private static class AssemblePartialAlignmentsParameters extends ActionParameter
names = {"-r", "--report"})
public String report;

@Parameter(description = "JSON report file.",
names = {"--json-report"})
public String jsonReport = null;

@Parameter(description = "Write only overlapped sequences (needed for testing).",
names = {"-o", "--overlapped-only"})
public Boolean overlappedOnly;
Expand Down

0 comments on commit a296012

Please sign in to comment.