Merge branch 'release/v1.3'

.gitignore

@@ -1,2 +1,4 @@
 mixcr.iml
 target
+*.iml
+.idea

CHANGELOG

@@ -1,4 +1,18 @@
 
+MiXCR 1.3 (13 Aug 2015)
+========================
+
+-- new feature: alignments with V and J genes from different loci are dropped (this behaviour can 
+be turned off by adding '--diff-loci' command line option)
+-- default parameter value changed: for cloneClusteringParameters.specificMutationProbability value 
+   nged from 1E-4 to 1E-3
+-- default parameter value changed: for cloneClusteringParameters.searchParameters value changed 
+   m `oneMismatch` to `twoMismatchesOrIndels`
+-- bug-fix: overlapping of reads now preserves read description
+-- bug-fix: `-descrR2` now outputs information from R2 (was from R1)
+-- minor bug-fixed: NPE in VDJCAlignerAbstract
+
+
 MiXCR v1.2 (16 Jun 2015)
 ========================
 

pom.xml

@@ -1,3 +1,4 @@
+<?xml version="1.0"?>
 <!--
   ~ Copyright (c) 2014-2015, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
   ~ (here and after addressed as Inventors)
@@ -26,13 +27,12 @@
   ~ PARTICULAR PURPOSE, OR THAT THE USE OF THE SOFTWARE WILL NOT INFRINGE ANY
   ~ PATENT, TRADEMARK OR OTHER RIGHTS.
   -->
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
     <groupId>com.milaboratory</groupId>
     <artifactId>mixcr</artifactId>
-    <version>1.2</version>
+    <version>1.3</version>
     <packaging>jar</packaging>
     <name>MiXCR</name>
 
@@ -42,23 +42,22 @@
         <version>7</version>
     </parent>
 
-    <dependencies>
-        <dependency>
-            <groupId>com.milaboratory</groupId>
-            <artifactId>milib</artifactId>
-            <version>1.0.1</version>
-        </dependency>
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <milib.version>1.0.2</milib.version>
+    </properties>
 
+    <dependencies>
         <dependency>
             <groupId>com.milaboratory</groupId>
             <artifactId>mitools</artifactId>
-            <version>1.0.1</version>
+            <version>${milib.version}</version>
         </dependency>
 
         <dependency>
             <groupId>com.milaboratory</groupId>
             <artifactId>milib</artifactId>
-            <version>1.0.1</version>
+            <version>${milib.version}</version>
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
@@ -96,10 +95,6 @@
         </dependency>
     </dependencies>
 
-    <properties>
-        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    </properties>
-
     <build>
         <plugins>
             <plugin>
@@ -162,15 +157,12 @@
                                 </filter>
                             </filters>
                             <transformers>
-                                <transformer
-                                        implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer">
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer">
                                 </transformer>
-                                <transformer
-                                        implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
                                     <addHeader>false</addHeader>
                                 </transformer>
-                                <transformer
-                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                     <mainClass>com.milaboratory.mixcr.cli.Main</mainClass>
                                 </transformer>
                             </transformers>

src/main/java/com/milaboratory/mixcr/basictypes/PartitionedSequenceCached.java

@@ -41,7 +41,7 @@ public abstract class PartitionedSequenceCached<S extends Seq<S>> extends Partit
     final ConcurrentHashMap<GeneFeature, Object> cache = new ConcurrentHashMap<>();
 
     @Override
-    public S getFeature(GeneFeature feature) {
+    public synchronized S getFeature(GeneFeature feature) {
         Object seq;
         // (IMPORTANT) Exactly the same reference must be returned for the same input for correct serialization/deserialization
         if ((seq = cache.get(feature)) == null && !cache.containsKey(feature))

src/main/java/com/milaboratory/mixcr/basictypes/VDJCAlignments.java

@@ -76,6 +76,24 @@ public String[] getDescriptions() {
         return descriptions;
     }
 
+    /**
+     * Returns {@code true} if at least ont V and one J hit among first {@code top} hits have same locus and false
+     * otherwise (first {@code top} V hits have different locus from those have first {@code top} J hits).
+     *
+     * @param top numer of top hits to test
+     * @return {@code true} if at least ont V and one J hit among first {@code top} hits have same locus and false
+     * otherwise (first {@code top} V hits have different locus from those have first {@code top} J hits)
+     */
+    public final boolean hasSameVJLoci(final int top) {
+        VDJCHit[] vHits = hits.get(GeneType.Variable),
+                jHits = hits.get(GeneType.Joining);
+        for (int v = 0; v < top && v < vHits.length; ++v)
+            for (int j = 0; j < top && j < jHits.length; ++j)
+                if (vHits[v].getAllele().getLocus() == jHits[j].getAllele().getLocus())
+                    return true;
+        return false;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

src/main/java/com/milaboratory/mixcr/basictypes/VDJCAlignmentsReader.java

@@ -86,12 +86,12 @@ public void init() {
         this.usedAlleles = IOUtil.readAlleleReferences(input, alleleResolver, parameters);
     }
 
-    public VDJCAlignerParameters getParameters() {
+    public synchronized VDJCAlignerParameters getParameters() {
         init();
         return parameters;
     }
 
-    public List<Allele> getUsedAlleles() {
+    public synchronized List<Allele> getUsedAlleles() {
         init();
         return usedAlleles;
     }
@@ -113,7 +113,7 @@ public boolean isFinished() {
     }
 
     @Override
-    public void close() {
+    public synchronized void close() {
         close(false);
     }
 
@@ -134,7 +134,7 @@ private void close(boolean onEnd) {
     }
 
     @Override
-    public VDJCAlignments take() {
+    public synchronized VDJCAlignments take() {
         if (closed)
             return null;
 

src/main/java/com/milaboratory/mixcr/cli/ActionAlign.java

@@ -89,8 +89,10 @@ public void go(ActionHelper helper) throws Exception {
 //                    System.err.println("WARNING: Functional allele excluded " + allele.getName());
 
         AlignerReport report = actionParameters.report == null ? null : new AlignerReport();
-        if (report != null)
+        if (report != null) {
             aligner.setEventsListener(report);
+            report.setAllowDifferentVJLoci(actionParameters.allowDifferentVJLoci);
+        }
 
         try (SequenceReaderCloseable<? extends SequenceRead> reader = actionParameters.createReader();
              VDJCAlignmentsWriter writer = actionParameters.getOutputName().equals(".") ? null : new VDJCAlignmentsWriter(actionParameters.getOutputName())) {
@@ -111,6 +113,12 @@ public long getIndex(VDJCAlignmentResult o) {
                             }))) {
                 if (result.alignment == null)
                     continue;
+                if (!result.alignment.hasSameVJLoci(1)) {
+                    if (report != null)
+                        report.onAlignmentWithDifferentVJLoci();
+                    if (!actionParameters.allowDifferentVJLoci)
+                        continue;
+                }
                 if (writer != null) {
                     if (actionParameters.saveReadDescription)
                         result.alignment.setDescriptions(extractDescription(result.read));
@@ -150,7 +158,7 @@ public static class AlignParameters extends ActionParametersWithOutput {
         public List<String> parameters = new ArrayList<>();
 
         @DynamicParameter(names = "-O", description = "Overrides base values of parameters.")
-        private Map<String, String> overrides = new HashMap<>();
+        public Map<String, String> overrides = new HashMap<>();
 
         @Parameter(description = "Parameters",
                 names = {"-p", "--parameters"})
@@ -189,6 +197,10 @@ public static class AlignParameters extends ActionParametersWithOutput {
                 names = {"-a", "--save-description"})
         public Boolean saveReadDescription = false;
 
+        @Parameter(description = "Allow alignments with different loci of V and J hits.",
+                names = {"-i", "--diff-loci"})
+        public Boolean allowDifferentVJLoci = false;
+
         public int getTaxonID() {
             return Species.fromStringStrict(species);
         }

src/main/java/com/milaboratory/mixcr/cli/ActionAlignmentsStat.java

@@ -29,6 +29,7 @@
 package com.milaboratory.mixcr.cli;
 
 import cc.redberry.pipe.CUtils;
+import cc.redberry.pipe.VoidProcessor;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.ParameterException;
 import com.beust.jcommander.Parameters;
@@ -38,9 +39,13 @@
 import com.milaboratory.mitools.cli.HiddenAction;
 import com.milaboratory.mixcr.basictypes.VDJCAlignments;
 import com.milaboratory.mixcr.basictypes.VDJCAlignmentsReader;
+import com.milaboratory.mixcr.info.AlignmentInfoCollector;
+import com.milaboratory.mixcr.info.GeneFeatureCoverageCollector;
+import com.milaboratory.mixcr.info.ReferencePointCoverageCollector;
 import com.milaboratory.mixcr.reference.GeneFeature;
 import com.milaboratory.mixcr.reference.LociLibraryManager;
 import com.milaboratory.mixcr.reference.ReferencePoint;
+import com.milaboratory.util.SmartProgressReporter;
 
 import java.io.BufferedOutputStream;
 import java.io.FileOutputStream;
@@ -49,38 +54,54 @@
 import java.util.List;
 
 import static com.milaboratory.mixcr.reference.GeneFeature.*;
+import static com.milaboratory.mixcr.reference.ReferencePoint.*;
 
 @HiddenAction
 public class ActionAlignmentsStat implements Action {
     final AParameters actionParameters = new AParameters();
 
     @Override
     public void go(ActionHelper helper) throws Exception {
+
         long[] geneFeatureCounters = new long[targetFeatures.length];
+        AlignmentInfoCollector[] collectors = new AlignmentInfoCollector[targetFeatures.length + targetReferencePoints.length];
+
+        int i = 0;
+
+        for (GeneFeature targetFeature : targetFeatures)
+            collectors[i++] = new GeneFeatureCoverageCollector(targetFeature);
+
+        for (ReferencePoint targetReferencePoint : targetReferencePoints)
+            collectors[i++] = new ReferencePointCoverageCollector(targetReferencePoint, 40, 40);
+
+        final Collector collector = new Collector(collectors);
+
         try (VDJCAlignmentsReader reader = new VDJCAlignmentsReader(actionParameters.getInputFileName(),
                 LociLibraryManager.getDefault());
              PrintStream output = actionParameters.getOutputFileName().equals("-") ? System.out :
                      new PrintStream(new BufferedOutputStream(new FileOutputStream(actionParameters.getOutputFileName()), 32768))
         ) {
-            long totalCount = 0;
-            for (VDJCAlignments alignments : CUtils.it(reader)) {
-                ++totalCount;
-                for (int i = 0; i < geneFeatureCounters.length; i++)
-                    if (alignments.getFeature(targetFeatures[i]) != null)
-                        geneFeatureCounters[i]++;
-            }
-
-            for (int i = 0; i < geneFeatureCounters.length; i++) {
-                output.println("" + GeneFeature.encode(targetFeatures[i]) + ": " + geneFeatureCounters[i] +
-                        " (" + Util.PERCENT_FORMAT.format(100.0 * geneFeatureCounters[i] / totalCount) + "%)");
-            }
+            SmartProgressReporter.startProgressReport("Analysis", reader);
+            CUtils.processAllInParallel(reader, collector, Math.min(4, Runtime.getRuntime().availableProcessors()));
+            collector.end();
+
+            if (output == System.out)
+                output.println();
+
+            collector.write(output);
         }
     }
 
     private static final GeneFeature[] targetFeatures = {
             V5UTR,
-            new GeneFeature(ReferencePoint.L1Begin, -20, 0),
-            L1, Intron, L2, FR1, CDR1, FR2, CDR2, FR3, CDR3, FR4
+            new GeneFeature(L1Begin, -20, 0),
+            L1, Intron, L2, FR1, CDR1, FR2, CDR2, FR3, CDR3, FR4,
+            new GeneFeature(FR4, 0, -3)
+    };
+
+    private static final ReferencePoint[] targetReferencePoints = {
+            L1Begin, L1End, L2Begin, FR1Begin, CDR1Begin, FR2Begin, CDR2Begin, FR3Begin, CDR3Begin,
+            FR4Begin, FR4End
     };
 
     @Override
@@ -113,4 +134,28 @@ public void validate() {
             super.validate();
         }
     }
+
+    private static class Collector implements VoidProcessor<VDJCAlignments> {
+        final AlignmentInfoCollector[] collectors;
+
+        public Collector(AlignmentInfoCollector... collectors) {
+            this.collectors = collectors;
+        }
+
+        @Override
+        public void process(VDJCAlignments input) {
+            for (AlignmentInfoCollector collector : collectors)
+                collector.put(input);
+        }
+
+        public void end() {
+            for (AlignmentInfoCollector collector : collectors)
+                collector.end();
+        }
+
+        public void write(PrintStream writer) {
+            for (AlignmentInfoCollector collector : collectors)
+                collector.writeResult(writer);
+        }
+    }
 }

src/main/java/com/milaboratory/mixcr/cli/AlignerReport.java

@@ -39,8 +39,10 @@
 public final class AlignerReport implements VDJCAlignerEventListener, ReportWriter {
     private final AtomicLongArray fails = new AtomicLongArray(VDJCAlignmentFailCause.values().length);
     private final AtomicLong successes = new AtomicLong(0);
+    private final AtomicLong hasDifferentVJLoci = new AtomicLong(0);
     private final AtomicLong alignedOverlap = new AtomicLong(0);
     private final AtomicLong nonAlignedOverlap = new AtomicLong(0);
+    private volatile boolean allowDifferentVJLoci = false;
 
     public long getFails(VDJCAlignmentFailCause cause) {
         return fails.get(cause.ordinal());
@@ -70,6 +72,10 @@ public long getNonAlignedOverlap() {
         return nonAlignedOverlap.get();
     }
 
+    public void setAllowDifferentVJLoci(boolean allowDifferentVJLoci) {
+        this.allowDifferentVJLoci = allowDifferentVJLoci;
+    }
+
     @Override
     public void onFailedAlignment(SequenceRead read, VDJCAlignmentFailCause cause) {
         fails.incrementAndGet(cause.ordinal());
@@ -88,12 +94,21 @@ public void onSuccessfulOverlap(SequenceRead read, VDJCAlignments alignments) {
             alignedOverlap.incrementAndGet();
     }
 
+    public void onAlignmentWithDifferentVJLoci() {
+        hasDifferentVJLoci.incrementAndGet();
+    }
+
     @Override
     public void writeReport(ReportHelper helper) {
         long total = getTotal();
+        long success = allowDifferentVJLoci ? successes.get() : successes.get() - hasDifferentVJLoci.get();
         helper.writeField("Total sequencing reads", total);
-        helper.writeField("Successfully aligned reads", successes.get());
-        helper.writePercentField("Successfully aligned, percent", successes.get(), total);
+        helper.writeField("Successfully aligned reads", success);
+        helper.writePercentField("Successfully aligned, percent", success, total);
+        helper.writePercentField(allowDifferentVJLoci ?
+                        "Alignment with different V and J loci" :
+                        "Alignment filtered because of different V and J loci",
+                hasDifferentVJLoci.get(), total);
         for (VDJCAlignmentFailCause cause : VDJCAlignmentFailCause.values())
             helper.writePercentField("Alignment failed because of " + cause.name,
                     fails.get(cause.ordinal()), total);

src/main/java/com/milaboratory/mixcr/export/FieldExtractors.java

@@ -270,7 +270,7 @@ protected String extract(VDJCAlignments object) {
                             throw new IllegalArgumentException("Error for option \'-descrR2\':\n" +
                                     "No description available for second read: either re-run align action with --save-description option " +
                                     "or don't use \'-descrR2\' in exportAlignments");
-                        return ds[0];
+                        return ds[1];
                     }
                 });
             }