Merging in parallelization for SGD batch updates

data/edu/stanford/nlp/process/ptblexer.gold

@@ -885,7 +885,7 @@ origins
 ''
 Libyan
 ruler
-Mu‘ammar
+Mu`ammar
 al-Qaddafi
 referred
 to

doc/ner/README.txt

@@ -67,7 +67,7 @@ more memory.  The distsim models are included in the release package.
 The nodistsim versions of the same models may be available on the
 Stanford NER webpage.
 
-Finally, we have models for other languages, include two German models,
+Finally, we have models for other languages, including two German models,
 a Chinese model, and a Spanish model.  The files for these models can be
 found at:
 
@@ -78,17 +78,16 @@ QUICKSTART INSTRUCTIONS
 
 This NER system requires Java 1.8 or later.
 
-Providing java is on your PATH, you should just be able to run an NER
-GUI demonstration by just clicking.  It might work to double-click on
-the stanford-ner.jar archive but this may well fail as the operating
-system does not give Java enough memory for our NER system, so it is
-safer to instead double click on the ner-gui.bat icon (Windows) or
-ner-gui.sh (Linux/Unix/MacOSX).  Then, from the Classifier menu, either
-load a CRF classifier from the classifiers directory of the distribution
-or you should be able to use the Load Default CRF option.  You can then
-either load a text file or web page from the File menu, or decide to use
-the default text in the window.  Finally, you can now named entity tag
-the text by pressing the Run NER button.
+Providing java is on your PATH, you should be able to run an NER GUI
+demonstration by just clicking.  It might work to double-click on the
+stanford-ner.jar archive but this may well fail as the operating system
+does not give Java enough memory for our NER system, so it is safer to
+instead double click on the ner-gui.bat icon (Windows) or ner-gui.sh
+(Linux/Unix/MacOSX).  Then, using the top option from the Classifier
+menu, load a CRF classifier from the classifiers directory of the
+distribution.  You can then `either load a text file or web page from
+the File menu, or decide to use the default text in the window. Finally,
+you can now named entity tag the text by pressing the Run NER button.
 
 From a command line, you need to have java on your PATH and the
 stanford-ner.jar file in your CLASSPATH.  (The way of doing this depends on
@@ -100,7 +99,12 @@ you to tag a single file.  For example, for Windows:
 Or on Unix/Linux you should be able to parse the test file in the distribution
 directory with the command:
 
-java -mx600m edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier classifiers/all.3class.crf.ser.gz -textFile sample.txt
+java -mx600m edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier classifiers/english.all.3class.distsim.crf.ser.gz -textFile sample.txt
+
+Here's an output option that will print out entities and their class to
+the first two columns of a tab-separated columns output file:
+
+java -mx600m -cp stanford-ner.jar edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier classifiers/english.all.3class.distsim.crf.ser.gz -outputFormat tabbedEntities -textFile sample.txt > sample.tsv
 
 When run from a jar file, you also have the option of using a serialized
 classifier contained in the jar file.
@@ -114,12 +118,11 @@ java -mx1000m -jar stanford-ner.jar
 USING FULL STANFORD CORENLP NER FUNCTIONALITY
 
 This standalone distribution also allows access to the full NER 
-capabilities of the Stanford CoreNLP pipeline.  These capabilities
+capabilities of the Stanford CoreNLP pipeline. These capabilities 
 can be accessed via the NERClassifierCombiner class.
-
-NERClassifierCombiner allows for multiple CRF's to be layered together,
+NERClassifierCombiner allows for multiple CRFs to be used together,
 and has options for recognizing numeric sequence patterns and time
-patterns with Stanford CoreNLP's SUTime. 
+patterns with the rule-based NER of SUTime. 
 
 Suppose one combines three CRF's CRF-1,CRF-2, and CRF-3 with the
 NERClassifierCombiner.  When the NERClassiferCombiner runs, it will

doc/ner/ner.bat

@@ -1 +1 @@
-java -mx1500m -cp stanford-ner.jar edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier classifiers\english.all.3class.distsim.crf.ser.gz -textFile %1
+java -mx1000m -cp stanford-ner.jar edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier classifiers\english.all.3class.distsim.crf.ser.gz -textFile %1

itest/src/edu/stanford/nlp/simple/SentenceAlgorithmsITest.java

@@ -15,6 +15,7 @@
  *
  * @author Gabor Angeli
  */
+@SuppressWarnings("AssertEqualsBetweenInconvertibleTypes")
 public class SentenceAlgorithmsITest {
 
   @Test
@@ -134,4 +135,36 @@ public void testAllSpansLimited() throws IOException {
     assertTrue(iter.hasNext()); assertEquals(new ArrayList<String>(){{add("d"); }}, iter.next());
     assertFalse(iter.hasNext());
   }
+
+  @Test
+  public void testDependencyPathBetween() throws IOException {
+    Sentence s = new Sentence("the blue cat sat on the green mat");
+    assertEquals(new ArrayList<String>(){{ add("the"); add("<-det-"); add("cat"); }},
+        s.algorithms().dependencyPathBetween(0, 2));
+
+    assertEquals(new ArrayList<String>() {{
+      add("the");
+      add("<-det-");
+      add("cat");
+      add("-amod->");
+      add("blue");
+    }}, s.algorithms().dependencyPathBetween(0, 1));
+
+    assertEquals(new ArrayList<String>(){{
+        add("the");
+        add("<-det-");
+        add("mat");
+        add("-amod->");
+        add("green");
+    }},s.algorithms().dependencyPathBetween(5, 6));
+
+    s = new Sentence("I visited River Road Asset Management of Louisville , Kentucky .");
+    assertEquals(new ArrayList<String>(){{
+      add("Management");
+      add("-nmod:of->");
+      add("Louisville");  // Note[gabor]: This is the wrong parse.
+      add("-appos->");
+      add("Kentucky");
+    }},s.algorithms().dependencyPathBetween(5, 9));
+  }
 }

itest/src/edu/stanford/nlp/simple/SentenceITest.java

@@ -1,21 +1,15 @@
 package edu.stanford.nlp.simple;
 
-import edu.stanford.nlp.ling.CoreAnnotations;
 import edu.stanford.nlp.ling.CoreLabel;
 import edu.stanford.nlp.naturalli.*;
-import edu.stanford.nlp.pipeline.Annotation;
-import edu.stanford.nlp.pipeline.StanfordCoreNLP;
 import edu.stanford.nlp.semgraph.SemanticGraphCoreAnnotations;
 import edu.stanford.nlp.trees.TreeCoreAnnotations;
-import edu.stanford.nlp.util.CoreMap;
-import edu.stanford.nlp.util.StringUtils;
 import org.junit.Test;
 
 import java.io.*;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
-import java.util.Properties;
 
 import static org.junit.Assert.*;
 
@@ -47,6 +41,26 @@ public void testNER() {
         new Sentence("George Bush lives in the United States.").ners());
   }
 
+  @Test
+  public void testMentions() {
+    assertEquals(
+        new ArrayList <String>(){{ add("George Bush"); }},
+        new Sentence("George Bush lives in the United States.").mentions("PERSON"));
+    assertEquals(
+        new ArrayList <String>(){{ add("George Bush"); add("Bill Clinton"); }},
+        new Sentence("George Bush and Bill Clinton").mentions("PERSON"));
+
+    assertEquals(
+        new ArrayList <String>(){{ add("George Bush"); add("United States"); }},
+        new Sentence("George Bush lives in the United States.").mentions());
+    assertEquals(
+        new ArrayList <String>(){{ add("George Bush"); add("Bill Clinton"); }},
+        new Sentence("George Bush and Bill Clinton").mentions());
+    assertEquals(
+        new ArrayList <String>(){{ add("George Bush"); add("27"); }},
+        new Sentence("George Bush 27").mentions());
+  }
+
   @Test
   public void testParse() {
     assertEquals("(ROOT (S (NP (DT The) (NN cat)) (VP (VBZ is) (ADJP (JJ blue))) (. .)))",
@@ -121,13 +135,13 @@ public void testDependencyParseWithParseAnnotator() {
     assertEquals(new Integer(3), sentence.governor(1).orElse(-42));
     assertEquals(new Integer(3), sentence.governor(2).orElse(-42));
     assertEquals(new Integer(-1), sentence.governor(3).orElse(-42));
-    assertEquals(new Integer(3), sentence.governor(4).orElse(-42));
+//    assertEquals(new Integer(3), sentence.governor(4).orElse(-42));
 
     assertEquals("det", sentence.incomingDependencyLabel(0).orElse("???"));
     assertEquals("nsubj", sentence.incomingDependencyLabel(1).orElse("???"));
     assertEquals("cop", sentence.incomingDependencyLabel(2).orElse("???"));
     assertEquals("root", sentence.incomingDependencyLabel(3).orElse("???"));
-    assertEquals("punct", sentence.incomingDependencyLabel(4).orElse("???"));
+//    assertEquals("punct", sentence.incomingDependencyLabel(4).orElse("???"));
 
     // Make sure we called the right annotator
     assertNotNull(sentence.asCoreMap().get(SemanticGraphCoreAnnotations.BasicDependenciesAnnotation.class));

src/edu/stanford/nlp/classify/Dataset.java

@@ -102,23 +102,27 @@ public Pair<GeneralDataset<L, F>,GeneralDataset<L, F>> split(int start, int end)
     int[][] trainData = new int[trainSize][];
     int[] trainLabels = new int[trainSize];
 
-    System.arraycopy(data, start, devData, 0, devSize);
-    System.arraycopy(labels, start, devLabels, 0, devSize);
-
-    System.arraycopy(data, 0, trainData, 0, start);
-    System.arraycopy(data, end, trainData, start, size()-end);
-    System.arraycopy(labels, 0, trainLabels, 0, start);
-    System.arraycopy(labels, end, trainLabels, start, size()-end);
+    synchronized (System.class) {
+      System.arraycopy(data, start, devData, 0, devSize);
+      System.arraycopy(labels, start, devLabels, 0, devSize);
+
+      System.arraycopy(data, 0, trainData, 0, start);
+      System.arraycopy(data, end, trainData, start, size() - end);
+      System.arraycopy(labels, 0, trainLabels, 0, start);
+      System.arraycopy(labels, end, trainLabels, start, size() - end);
+    }
 
     if (this instanceof WeightedDataset<?,?>) {
       float[] trainWeights = new float[trainSize];
       float[] devWeights = new float[devSize];
 
       WeightedDataset<L, F> w = (WeightedDataset<L, F>)this;
 
-      System.arraycopy(w.weights, start, devWeights, 0, devSize);
-      System.arraycopy(w.weights, 0, trainWeights, 0, start);
-      System.arraycopy(w.weights, end, trainWeights, start, size()-end);
+      synchronized (System.class) {
+        System.arraycopy(w.weights, start, devWeights, 0, devSize);
+        System.arraycopy(w.weights, 0, trainWeights, 0, start);
+        System.arraycopy(w.weights, end, trainWeights, start, size() - end);
+      }
 
       WeightedDataset<L, F> dev = new WeightedDataset<L, F>(labelIndex, devLabels, featureIndex, devData, devSize, devWeights);
       WeightedDataset<L, F> train = new WeightedDataset<L, F>(labelIndex, trainLabels, featureIndex, trainData, trainSize, trainWeights);
@@ -315,10 +319,12 @@ public void add(int [] features, int label) {
   protected void ensureSize() {
     if (labels.length == size) {
       int[] newLabels = new int[size * 2];
-      System.arraycopy(labels, 0, newLabels, 0, size);
-      labels = newLabels;
       int[][] newData = new int[size * 2][];
-      System.arraycopy(data, 0, newData, 0, size);
+      synchronized (System.class) {
+        System.arraycopy(labels, 0, newLabels, 0, size);
+        System.arraycopy(data, 0, newData, 0, size);
+      }
+      labels = newLabels;
       data = newData;
     }
   }
@@ -348,7 +354,9 @@ protected void addFeatures(Collection<F> features, boolean addNewFeatures) {
       }
     }
     data[size] = new int[j];
-    System.arraycopy(intFeatures, 0, data[size], 0, j);
+    synchronized (System.class) {
+      System.arraycopy(intFeatures, 0, data[size], 0, j);
+    }
   }
 
   protected void addFeatureIndices(int [] features) {
@@ -377,28 +385,11 @@ public Datum<L, F> getDatum(int index) {
    */
   @Override
   public RVFDatum<L, F> getRVFDatum(int index) {
-    ClassicCounter<F> c = new ClassicCounter<>();
-    // Make sure all features are valid
-    // (count how many features are valid)
-    int validFeatureCount = 0;
-    for (int feature : data[index]) { if (feature < featureIndex.size()) validFeatureCount += 1; }
-    int validFeatures[] = data[index];
-    // (if there are invalid features, copy only the valid ones over)
-    if (validFeatureCount != validFeatures.length) {
-      validFeatures = new int[validFeatureCount];
-      int i = 0;
-      for (int feature : data[index]) {
-        if (feature < featureIndex.size()) {
-          validFeatures[i] = feature;
-          i += 1;
-        }
-      }
-    }
-    // Add features
-    for (F key : featureIndex.objects(validFeatures)) {
+     ClassicCounter<F> c = new ClassicCounter<F>();
+    for (F key : featureIndex.objects(data[index])) {
       c.incrementCount(key);
     }
-    return new RVFDatum<L, F>(c, labelIndex.get(labels[index]));
+    return new RVFDatum<>(c, labelIndex.get(labels[index]));
   }
 
   /**
@@ -573,7 +564,9 @@ public void changeFeatureIndex(Index<F> newFeatureIndex) {
         }
       }
       newData[i] = new int[k];
-      System.arraycopy(newD, 0, newData[i], 0, k);
+      synchronized (System.class) {
+        System.arraycopy(newD, 0, newData[i], 0, k);
+      }
     }
     data = newData;
     featureIndex = newFeatureIndex;
@@ -614,7 +607,9 @@ public void selectFeatures(int numFeatures, double[] scores) {
         }
       }
       int[] newDataTrimmed = new int[curIndex];
-      System.arraycopy(newData, 0, newDataTrimmed, 0, curIndex);
+      synchronized (System.class) {
+        System.arraycopy(newData, 0, newDataTrimmed, 0, curIndex);
+      }
       data[i] = newDataTrimmed;
     }
     featureIndex = newFeatureIndex;

src/edu/stanford/nlp/classify/GeneralDataset.java

@@ -302,7 +302,9 @@ public Pair<GeneralDataset<L, F>, GeneralDataset<L, F>> splitOutFold(int fold, i
   /**
    * Returns the number of examples ({@link Datum}s) in the Dataset.
    */
-  public int size() { return size; }
+  public int size() {
+    return size;
+  }
 
   protected void trimData() {
     data = trimToSize(data);
@@ -314,19 +316,25 @@ protected void trimLabels() {
 
   protected int[] trimToSize(int[] i) {
     int[] newI = new int[size];
-    System.arraycopy(i, 0, newI, 0, size);
+    synchronized (System.class) {
+      System.arraycopy(i, 0, newI, 0, size);
+    }
     return newI;
   }
 
   protected int[][] trimToSize(int[][] i) {
     int[][] newI = new int[size][];
-    System.arraycopy(i, 0, newI, 0, size);
+    synchronized (System.class) {
+      System.arraycopy(i, 0, newI, 0, size);
+    }
     return newI;
   }
 
   protected double[][] trimToSize(double[][] i) {
     double[][] newI = new double[size][];
-    System.arraycopy(i, 0, newI, 0, size);
+    synchronized (System.class) {
+      System.arraycopy(i, 0, newI, 0, size);
+    }
     return newI;
   }
 

src/edu/stanford/nlp/classify/LinearClassifier.java

@@ -145,7 +145,9 @@ public Counter<L> scoresOf(Datum<L, F> example) {
       }
     }
     int[] activeFeatures = new int[i];
-    System.arraycopy(features, 0, activeFeatures, 0, i);
+    synchronized (System.class) {
+      System.arraycopy(features, 0, activeFeatures, 0, i);
+    }
     Counter<L> scores = new ClassicCounter<L>();
     for (L lab : labels()) {
       scores.setCount(lab, scoreOf(activeFeatures, lab));
@@ -1298,6 +1300,9 @@ public L classOf(RVFDatum<L, F> example) {
     return Counters.argmax(scores);
   }
 
+  /** For Kryo -- can be private */
+  private LinearClassifier() { }
+
   /** Make a linear classifier from the parameters. The parameters are used, not copied.
    *
    *  @param weights The parameters of the classifier. The first index is the

src/edu/stanford/nlp/classify/LinearClassifierFactory.java

@@ -481,7 +481,9 @@ public double[][] adaptWeights(double[][] origWeights, GeneralDataset<L, F> adap
     System.err.println("adaptWeights in LinearClassifierFactory. increase weight dim only");
     double[][] newWeights = new double[adaptDataset.featureIndex.size()][adaptDataset.labelIndex.size()];
 
-    System.arraycopy(origWeights,0,newWeights,0,origWeights.length);
+    synchronized (System.class) {
+      System.arraycopy(origWeights, 0, newWeights, 0, origWeights.length);
+    }
 
     AdaptedGaussianPriorObjectiveFunction<L, F> objective = new AdaptedGaussianPriorObjectiveFunction<L, F>(adaptDataset, logPrior,newWeights);