Change signature of printProbsDocument to return a Pair of Counters.

src/edu/stanford/nlp/ie/AbstractSequenceClassifier.java

@@ -44,6 +44,7 @@
 import edu.stanford.nlp.stats.Counter;
 import edu.stanford.nlp.stats.Counters;
 import edu.stanford.nlp.stats.Sampler;
+import edu.stanford.nlp.stats.TwoDimensionalCounter;
 import edu.stanford.nlp.util.*;
 import edu.stanford.nlp.util.concurrent.*;
 
@@ -949,10 +950,41 @@ public void printProbs(String filename,
    *          {@link CoreMap}.
    */
   public void printProbsDocuments(ObjectBank<List<IN>> documents) {
+    Counter<Integer> calibration = new ClassicCounter<>();
+    TwoDimensionalCounter<Integer,String> calibratedTokens = new TwoDimensionalCounter<>();
+
     for (List<IN> doc : documents) {
-      printProbsDocument(doc);
+      Pair<Counter<Integer>, TwoDimensionalCounter<Integer,String>> pair = printProbsDocument(doc);
+      if (pair != null) {
+        Counters.addInPlace(calibration, pair.first());
+        calibratedTokens.addAll(pair.second());
+      }
       System.out.println();
     }
+    if (calibration.size() > 0) {
+      // we stored stuff, so print it out
+      PrintWriter pw = new PrintWriter(System.err);
+      outputCalibrationInfo(pw, calibration, calibratedTokens);
+      pw.flush();
+    }
+  }
+
+  public static void outputCalibrationInfo(PrintWriter pw,
+                                           Counter<Integer> calibration,
+                                           TwoDimensionalCounter<Integer,String> calibratedTokens) {
+    final int numBins = 10;
+    pw.println("----------------------------------------");
+    pw.println("Probability distribution given to tokens");
+    pw.println("----------------------------------------");
+    for (int i = 0; i < numBins; i++) {
+      pw.printf("[%.1f-%.1f%c: %.1f  %s%n",
+              ((double) i) / numBins,
+              ((double) (i+1)) / numBins,
+              i == (numBins - 1) ? ']': ')',
+              calibration.getCount(i),
+              Counters.toSortedString(calibratedTokens.getCounter(i), 10, "%s=%.1f", ", ", "[%s]"));
+    }
+    pw.println("----------------------------------------");
   }
 
   public void classifyStdin()
@@ -974,7 +1006,7 @@ public void classifyStdin(DocumentReaderAndWriter<IN> readerWriter)
     }
   }
 
-  public abstract void printProbsDocument(List<IN> document);
+  public abstract Pair<Counter<Integer>, TwoDimensionalCounter<Integer,String>> printProbsDocument(List<IN> document);
 
   /**
    * Load a test file, run the classifier on it, and then print the answers to

src/edu/stanford/nlp/ie/ClassifierCombiner.java

@@ -6,9 +6,12 @@
 import edu.stanford.nlp.ling.HasWord;
 import edu.stanford.nlp.pipeline.DefaultPaths;
 import edu.stanford.nlp.sequences.DocumentReaderAndWriter;
+import edu.stanford.nlp.stats.Counter;
+import edu.stanford.nlp.stats.TwoDimensionalCounter;
 import edu.stanford.nlp.util.CoreMap;
 import edu.stanford.nlp.util.ErasureUtils;
 import edu.stanford.nlp.util.Generics;
+import edu.stanford.nlp.util.Pair;
 import edu.stanford.nlp.util.StringUtils;
 
 import java.io.FileNotFoundException;
@@ -395,7 +398,7 @@ public void train(Collection<List<IN>> docs,
   }
 
   @Override
-  public void printProbsDocument(List<IN> document) {
+  public Pair<Counter<Integer>, TwoDimensionalCounter<Integer,String>> printProbsDocument(List<IN> document) {
     throw new UnsupportedOperationException();
   }
 

src/edu/stanford/nlp/ie/crf/CRFClassifier.java

@@ -38,6 +38,7 @@
 import edu.stanford.nlp.sequences.*;
 import edu.stanford.nlp.stats.ClassicCounter;
 import edu.stanford.nlp.stats.Counter;
+import edu.stanford.nlp.stats.TwoDimensionalCounter;
 import edu.stanford.nlp.util.*;
 
 import java.io.*;
@@ -46,6 +47,7 @@
 import java.text.NumberFormat;
 import java.util.*;
 import java.util.regex.*;
+import java.util.stream.Collectors;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
@@ -1338,18 +1340,27 @@ public List<IN> classifyGibbs(List<IN> document, Triple<int[][][], int[], double
    * the likelihood of each possible label at each point.
    *
    * @param document A {@link List} of something that extends CoreMap.
+   * @return If verboseMode is set, a Pair of Counters recording classification decisions, else null.
    */
   @Override
-  public void printProbsDocument(List<IN> document) {
+  public Pair<Counter<Integer>, TwoDimensionalCounter<Integer,String>> printProbsDocument(List<IN> document) {
+    final int numBins = 10;
+    boolean verbose = flags.verboseMode;
 
     Triple<int[][][], int[], double[][][]> p = documentToDataAndLabels(document);
-
     CRFCliqueTree<String> cliqueTree = getCliqueTree(p);
 
+    Counter<Integer> calibration = new ClassicCounter<>();
+    TwoDimensionalCounter<Integer,String> calibratedTokens = new TwoDimensionalCounter<>();
+
     // for (int i = 0; i < factorTables.length; i++) {
     for (int i = 0; i < cliqueTree.length(); i++) {
       IN wi = document.get(i);
-      System.out.print(wi.get(CoreAnnotations.TextAnnotation.class));
+      String token = wi.get(CoreAnnotations.TextAnnotation.class);
+      String goldAnswer = wi.get(CoreAnnotations.GoldAnswerAnnotation.class);
+      System.out.print(token);
+      System.out.print('\t');
+      System.out.print(goldAnswer);
       for (String label : classIndex) {
         int index = classIndex.indexOf(label);
         // double prob = Math.pow(Math.E, factorTables[i].logProbEnd(index));
@@ -1358,9 +1369,24 @@ public void printProbsDocument(List<IN> document) {
         System.out.print(label);
         System.out.print('=');
         System.out.print(prob);
+        if (verbose ) {
+          int binnedProb = (int) prob * numBins;
+          if (binnedProb > (numBins - 1)) {
+            binnedProb = numBins - 1;
+          }
+          calibration.incrementCount(binnedProb);
+          if (label.equals(goldAnswer)) {
+            calibratedTokens.incrementCount(binnedProb, token);
+          }
+        }
       }
       System.out.println();
     }
+    if (verbose) {
+      return new Pair<>(calibration, calibratedTokens);
+    } else {
+      return null;
+    }
   }
 
   /**
@@ -1382,8 +1408,7 @@ public void printFirstOrderProbs(String filename, DocumentReaderAndWriter<IN> re
    * Takes a {@link List} of documents and prints the likelihood of each
    * possible label at each point.
    *
-   * @param documents
-   *          A {@link List} of {@link List} of INs.
+   * @param documents A {@link List} of {@link List} of INs.
    */
   public void printFirstOrderProbsDocuments(ObjectBank<List<IN>> documents) {
     for (List<IN> doc : documents) {
@@ -1395,8 +1420,7 @@ public void printFirstOrderProbsDocuments(ObjectBank<List<IN>> documents) {
   /**
    * Takes the file, reads it in, and prints out the factor table at each position.
    *
-   * @param filename
-   *          The path to the specified file
+   * @param filename The path to the specified file
    */
   public void printFactorTable(String filename, DocumentReaderAndWriter<IN> readerAndWriter) {
     // only for the OCR data does this matter
@@ -1410,8 +1434,7 @@ public void printFactorTable(String filename, DocumentReaderAndWriter<IN> reader
    * Takes a {@link List} of documents and prints the factor table
    * at each point.
    *
-   * @param documents
-   *          A {@link List} of {@link List} of INs.
+   * @param documents A {@link List} of {@link List} of INs.
    */
   public void printFactorTableDocuments(ObjectBank<List<IN>> documents) {
     for (List<IN> doc : documents) {
@@ -2964,10 +2987,7 @@ public static void main(String[] args) throws Exception {
     }
 
     if (testFiles != null) {
-      List<File> files = new ArrayList<File>();
-      for (String filename : testFiles.split(",")) {
-        files.add(new File(filename));
-      }
+      List<File> files = Arrays.asList(testFiles.split(",")).stream().map(File::new).collect(Collectors.toList());
       crf.classifyFilesAndWriteAnswers(files, crf.defaultReaderAndWriter(), true);
     }
 

src/edu/stanford/nlp/ie/ner/CMMClassifier.java

@@ -71,6 +71,7 @@
 import edu.stanford.nlp.sequences.SequenceModel;
 import edu.stanford.nlp.stats.ClassicCounter;
 import edu.stanford.nlp.stats.Counter;
+import edu.stanford.nlp.stats.TwoDimensionalCounter;
 import edu.stanford.nlp.util.CoreMap;
 import edu.stanford.nlp.util.ErasureUtils;
 import edu.stanford.nlp.util.Generics;
@@ -1547,15 +1548,14 @@ public Counter<String> scoresOf(List<IN> lineInfos, int pos) {
   /**
    * Takes a {@link List} of {@link CoreLabel}s and prints the likelihood
    * of each possible label at each point.
-   * TODO: Finish or delete this method!
+   * TODO: Write this method!
    *
    * @param document A {@link List} of {@link CoreLabel}s.
    */
   @Override
-  public void printProbsDocument(List<IN> document) {
-
+  public Pair<Counter<Integer>, TwoDimensionalCounter<Integer,String>> printProbsDocument(List<IN> document) {
     //ClassicCounter<String> c = scoresOf(document, 0);
-
+    throw new UnsupportedOperationException();
   }
 
   /** Command-line version of the classifier.  See the class

src/edu/stanford/nlp/ie/regexp/NumberSequenceClassifier.java

@@ -7,12 +7,15 @@
 import edu.stanford.nlp.pipeline.Annotation;
 import edu.stanford.nlp.sequences.DocumentReaderAndWriter;
 import edu.stanford.nlp.sequences.PlainTextDocumentReaderAndWriter;
+import edu.stanford.nlp.stats.Counter;
+import edu.stanford.nlp.stats.TwoDimensionalCounter;
 import edu.stanford.nlp.time.TimeAnnotations;
 import edu.stanford.nlp.time.TimeExpressionExtractor;
 import edu.stanford.nlp.time.TimeExpressionExtractorFactory;
 import edu.stanford.nlp.time.Timex;
 import edu.stanford.nlp.util.CoreMap;
 import edu.stanford.nlp.util.PaddedList;
+import edu.stanford.nlp.util.Pair;
 import edu.stanford.nlp.util.StringUtils;
 
 import java.io.ObjectInputStream;
@@ -809,7 +812,8 @@ public void train(Collection<List<CoreLabel>> docs,
   }
 
   @Override
-  public void printProbsDocument(List<CoreLabel> document) {
+  public Pair<Counter<Integer>, TwoDimensionalCounter<Integer,String>> printProbsDocument(List<CoreLabel> document) {
+    throw new UnsupportedOperationException();
   }
 
   @Override

src/edu/stanford/nlp/ie/regexp/RegexNERSequenceClassifier.java

@@ -22,8 +22,11 @@
 import edu.stanford.nlp.ling.CoreLabel;
 import edu.stanford.nlp.ling.CoreAnnotations;
 import edu.stanford.nlp.sequences.DocumentReaderAndWriter;
+import edu.stanford.nlp.stats.Counter;
+import edu.stanford.nlp.stats.TwoDimensionalCounter;
 import edu.stanford.nlp.util.CoreMap;
 import edu.stanford.nlp.util.Generics;
+import edu.stanford.nlp.util.Pair;
 
 /**
  * A sequence classifier that labels tokens with types based on a simple manual mapping from
@@ -357,7 +360,9 @@ public void train(Collection<List<CoreLabel>> docs,
                     DocumentReaderAndWriter<CoreLabel> readerAndWriter) {}
 
   @Override
-  public void printProbsDocument(List<CoreLabel> document) {}
+  public Pair<Counter<Integer>, TwoDimensionalCounter<Integer,String>> printProbsDocument(List<CoreLabel> document) {
+    throw new UnsupportedOperationException();
+  }
 
   @Override
   public void serializeClassifier(String serializePath) {}