Merge branch 'master' into mt-preordering-feat

itest/src/edu/stanford/nlp/parser/lexparser/LexicalizedParserITest.java

@@ -165,7 +165,7 @@ public void testParseString() {
                   "My/PRP$ dog/NN likes/VBZ to/TO eat/VB yoghurt/NN ./.",
                   "(ROOT (S (NP (PRP$ My) (NN dog)) (VP (VBZ likes) (S (VP (TO to) (VP (VB eat) (NP (NN yoghurt)))))) (. .)))",
                   "poss(dog-2, My-1) nsubj(likes-3, dog-2) root(ROOT-0, likes-3) aux(eat-5, to-4) xcomp(likes-3, eat-5) dobj(eat-5, yoghurt-6)",
-                  "poss(dog-2, My-1) nsubj(likes-3, dog-2) xsubj(eat-5, dog-2) root(ROOT-0, likes-3) aux(eat-5, to-4) xcomp(likes-3, eat-5) dobj(eat-5, yoghurt-6)");
+                  "poss(dog-2, My-1) nsubj(likes-3, dog-2) nsubj(eat-5, dog-2) root(ROOT-0, likes-3) aux(eat-5, to-4) xcomp(likes-3, eat-5) dobj(eat-5, yoghurt-6)");
   }
 
   /**

src/edu/stanford/nlp/dcoref/Mention.java

@@ -1367,7 +1367,7 @@ public String getRelation(){
     if(relation.toString().startsWith("prep") || relation == EnglishGrammaticalRelations.PREPOSITIONAL_OBJECT || relation == EnglishGrammaticalRelations.TEMPORAL_MODIFIER || relation == EnglishGrammaticalRelations.ADV_CLAUSE_MODIFIER || relation == EnglishGrammaticalRelations.ADVERBIAL_MODIFIER || relation == EnglishGrammaticalRelations.PREPOSITIONAL_COMPLEMENT) return "adjunct";
 
     // subject relations
-    if(relation == EnglishGrammaticalRelations.NOMINAL_SUBJECT || relation == EnglishGrammaticalRelations.CLAUSAL_SUBJECT || relation == EnglishGrammaticalRelations.CONTROLLING_SUBJECT) return "subject";
+    if(relation == EnglishGrammaticalRelations.NOMINAL_SUBJECT || relation == EnglishGrammaticalRelations.CLAUSAL_SUBJECT) return "subject";
     if(relation == EnglishGrammaticalRelations.NOMINAL_PASSIVE_SUBJECT || relation == EnglishGrammaticalRelations.CLAUSAL_PASSIVE_SUBJECT) return "subject";
 
     // verbal argument relations

src/edu/stanford/nlp/dcoref/sievepasses/DeterministicCorefSieve.java

@@ -477,16 +477,16 @@ private static List<Mention> sortMentionsForPronoun(List<Mention> l, Mention m1,
     if (sameSentence) {
       Tree tree = m1.contextParseTree;
       Tree current = m1.mentionSubTree;
-      while (true) {
-        current = current.ancestor(1, tree);
+      current = current.parent(tree);
+      while (current != null) {
         if (current.label().value().startsWith("S")) {
           for (Mention m : l) {
             if (!sorted.contains(m) && current.dominates(m.mentionSubTree)) {
               sorted.add(m);
             }
           }
         }
-        if (current.label().value().equals("ROOT") || current.ancestor(1, tree)==null) break;
+        current = current.parent(tree);
       }
       if (SieveCoreferenceSystem.logger.isLoggable(Level.FINEST)) {
         if (l.size()!=sorted.size()) {

src/edu/stanford/nlp/ie/NERFeatureFactory.java

@@ -1709,7 +1709,7 @@ protected Collection<String> featuresCpC(PaddedList<IN> cInfo, int loc) {
             featuresCpC.add(pWord + "-PSEQpW"); // added later after goodCoNLL
           }
 
-          if (true) { // TODO [cdm Jul 2014]: should really be if (flags.useDistSim) but fixing current itest....
+          if (flags.useDistSim) {
             featuresCpC.add(pDS + "-PSEQpDS");
             featuresCpC.add(cDS + "-PSEQcDS");
             featuresCpC.add(pDS+ '-' +cDS + "-PSEQpcDS");
@@ -1747,7 +1747,7 @@ protected Collection<String> featuresCpC(PaddedList<IN> cInfo, int loc) {
       if (flags.useTypeySequences) {
         featuresCpC.add(cShape + "-TPS2");
         featuresCpC.add(n.get(CoreAnnotations.ShapeAnnotation.class) + "-TNS1");
-        // featuresCpC.add(pShape) + "-" + cShape) + "-TPS"); // duplicates -TYPES, so now omitted; you may need to slighly increase sigma to duplicate previous results, however.
+        // featuresCpC.add(pShape) + "-" + cShape) + "-TPS"); // duplicates -TYPES, so now omitted; you may need to slightly increase sigma to duplicate previous results, however.
       }
 
       if (flags.useTaggySequences) {

src/edu/stanford/nlp/ie/machinereading/RelationFeatureFactory.java

@@ -929,9 +929,6 @@ private static GrammaticalRelation generalizeRelation(GrammaticalRelation gr) {
         return generalGR;
       }
     }
-    if (gr.equals(EnglishGrammaticalRelations.CONTROLLING_SUBJECT)) {
-      return EnglishGrammaticalRelations.SUBJECT;
-    }
     return gr;
   }
 

src/edu/stanford/nlp/io/IOUtils.java

@@ -453,12 +453,13 @@ public static InputStream getInputStreamFromURLOrClasspathOrFileSystem(String te
     }
 
     if (textFileOrUrl.endsWith(".gz")) {
-      // gunzip it if necessary. Since a GZIPInputStream has a buffer in it, don't need a second level of buffering
+      // gunzip it if necessary
       in = new GZIPInputStream(in, GZIP_FILE_BUFFER_SIZE);
-    } else {
-      // buffer this stream
-      in = new BufferedInputStream(in);
-    }
+    } 
+
+    // buffer this stream.  even gzip streams benefit from buffering,
+    // such as for the shift reduce parser
+    in = new BufferedInputStream(in);
 
     return in;
   }

src/edu/stanford/nlp/neural/SimpleTensor.java

@@ -18,7 +18,7 @@
  * @author Richard Socher
  */
 public class SimpleTensor implements Serializable {
-  private SimpleMatrix[] slices;
+  private final SimpleMatrix[] slices;
 
   final int numRows;
   final int numCols;
@@ -286,5 +286,15 @@ public void remove() {
     }
   }
 
+  @Override
+  public String toString() {
+    StringBuilder result = new StringBuilder();
+    for (int slice = 0; slice < numSlices; ++slice) {
+      result.append("Slice " + slice + "\n");
+      result.append(slices[slice]);
+    }
+    return result.toString();
+  }
+
   private static final long serialVersionUID = 1;
 }

src/edu/stanford/nlp/parser/common/ParserUtils.java

@@ -4,6 +4,7 @@
 import java.util.Collections;
 import java.util.List;
 
+import edu.stanford.nlp.ling.HasTag;
 import edu.stanford.nlp.ling.HasWord;
 import edu.stanford.nlp.trees.LabeledScoredTreeFactory;
 import edu.stanford.nlp.trees.Tree;
@@ -14,15 +15,21 @@ public class ParserUtils {
   /**
    * Construct a fall through tree in case we can't parse this sentence
    * @param words
-   * @return a tree with X for all the internal nodes
+   * @return a tree with X for all the internal nodes.  preterminals have the right tag if the words are tagged
    */
   public static Tree xTree(List<? extends HasWord> words) {
     TreeFactory lstf = new LabeledScoredTreeFactory();
     List<Tree> lst2 = new ArrayList<Tree>();
     for (HasWord obj : words) {
       String s = obj.word();
       Tree t = lstf.newLeaf(s);
-      Tree t2 = lstf.newTreeNode("X", Collections.singletonList(t));
+      String tag = "X";
+      if (obj instanceof HasTag) {
+        if (((HasTag) obj).tag() != null) {
+          tag = ((HasTag) obj).tag();
+        }
+      }
+      Tree t2 = lstf.newTreeNode(tag, Collections.singletonList(t));
       lst2.add(t2);
     }
     return lstf.newTreeNode("X", lst2);

src/edu/stanford/nlp/parser/shiftreduce/CompoundUnaryTransition.java

@@ -19,7 +19,10 @@
  * @author John Bauer
  */
 public class CompoundUnaryTransition implements Transition {
-  /** labels[0] is the top of the unary chain */
+  /**
+   * labels[0] is the top of the unary chain.  
+   * A unary chain that results in a ROOT will have labels[0] == ROOT, for example. 
+   */
   public final String[] labels;
 
   /** root transitions are illegal in the middle of the tree, naturally */

src/edu/stanford/nlp/parser/shiftreduce/CreateTransitionSequence.java

@@ -1,6 +1,7 @@
 package edu.stanford.nlp.parser.shiftreduce;
 
 import java.util.List;
+import java.util.Set;
 
 import edu.stanford.nlp.ling.CoreLabel;
 import edu.stanford.nlp.trees.Tree;
@@ -12,37 +13,38 @@ public class CreateTransitionSequence {
   // we could change this if we wanted to include options.
   private CreateTransitionSequence() {}
 
-  public static List<Transition> createTransitionSequence(Tree tree, boolean compoundUnary) {
+  public static List<Transition> createTransitionSequence(Tree tree, boolean compoundUnary, Set<String> rootStates, Set<String> rootOnlyStates) {
     List<Transition> transitions = Generics.newArrayList();
 
-    createTransitionSequenceHelper(transitions, tree, compoundUnary, true);
-    transitions.add(new FinalizeTransition());
+    createTransitionSequenceHelper(transitions, tree, compoundUnary, rootOnlyStates);
+    transitions.add(new FinalizeTransition(rootStates));
     transitions.add(new IdleTransition());
 
     return transitions;
   }
 
-  private static void createTransitionSequenceHelper(List<Transition> transitions, Tree tree, boolean compoundUnary, boolean isRoot) {
+  private static void createTransitionSequenceHelper(List<Transition> transitions, Tree tree, boolean compoundUnary, Set<String> rootOnlyStates) {
     if (tree.isLeaf()) {
       // do nothing
     } else if (tree.isPreTerminal()) {
       transitions.add(new ShiftTransition());
     } else if (tree.children().length == 1) {
+      boolean isRoot = rootOnlyStates.contains(tree.label().value());
       if (compoundUnary) {
         List<String> labels = Generics.newArrayList();
         while (tree.children().length == 1 && !tree.isPreTerminal()) {
           labels.add(tree.label().value());
           tree = tree.children()[0];
         }
-        createTransitionSequenceHelper(transitions, tree, compoundUnary, false);
+        createTransitionSequenceHelper(transitions, tree, compoundUnary, rootOnlyStates);
         transitions.add(new CompoundUnaryTransition(labels, isRoot));
       } else {
-        createTransitionSequenceHelper(transitions, tree.children()[0], compoundUnary, false);
+        createTransitionSequenceHelper(transitions, tree.children()[0], compoundUnary, rootOnlyStates);
         transitions.add(new UnaryTransition(tree.label().value(), isRoot));
       }
     } else if (tree.children().length == 2) {
-      createTransitionSequenceHelper(transitions, tree.children()[0], compoundUnary, false);
-      createTransitionSequenceHelper(transitions, tree.children()[1], compoundUnary, false);
+      createTransitionSequenceHelper(transitions, tree.children()[0], compoundUnary, rootOnlyStates);
+      createTransitionSequenceHelper(transitions, tree.children()[1], compoundUnary, rootOnlyStates);
 
       // This is the tricky part... need to decide if the binary
       // transition is a left or right transition.  This is done by

src/edu/stanford/nlp/parser/shiftreduce/FinalizeTransition.java

@@ -1,15 +1,38 @@
 package edu.stanford.nlp.parser.shiftreduce;
 
 import java.util.List;
+import java.util.Set;
 import edu.stanford.nlp.parser.common.ParserConstraint;
 import edu.stanford.nlp.trees.Tree;
 
+// only needed for readObject
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.util.Collections;
+import edu.stanford.nlp.util.ErasureUtils;
+
 /**
  * Transition that finishes the processing of a state
  */
 public class FinalizeTransition implements Transition {
+  private Set<String> rootStates;
+
+  private void readObject(ObjectInputStream in)
+    throws IOException, ClassNotFoundException 
+  {
+    ObjectInputStream.GetField fields = in.readFields();
+    rootStates = ErasureUtils.uncheckedCast(fields.get("rootStates", null));
+    if (rootStates == null) {
+      rootStates = Collections.singleton("ROOT");
+    }
+  }
+
+  public FinalizeTransition(Set<String> rootStates) {
+    this.rootStates = rootStates;
+  }
+
   public boolean isLegal(State state, List<ParserConstraint> constraints) {
-    boolean legal = !state.finished && state.tokenPosition >= state.sentence.size() && state.stack.size() == 1;
+    boolean legal = !state.finished && state.tokenPosition >= state.sentence.size() && state.stack.size() == 1 && rootStates.contains(state.stack.peek().value());
     if (!legal || constraints == null) {
       return legal;
     }
@@ -22,6 +45,7 @@ public boolean isLegal(State state, List<ParserConstraint> constraints) {
         return false;
       }
     }
+
     return true;
   }
 

src/edu/stanford/nlp/parser/shiftreduce/Oracle.java

@@ -4,6 +4,7 @@
 import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import edu.stanford.nlp.ling.CoreLabel;
 import edu.stanford.nlp.trees.Tree;
@@ -35,7 +36,9 @@ class Oracle {
 
   boolean compoundUnaries;
 
-  Oracle(List<Tree> binarizedTrees, boolean compoundUnaries) {
+  Set<String> rootStates;
+
+  Oracle(List<Tree> binarizedTrees, boolean compoundUnaries, Set<String> rootStates) {
     this.binarizedTrees = binarizedTrees;
 
     parentMaps = Generics.newArrayList(binarizedTrees.size());
@@ -136,7 +139,7 @@ OracleTransition goldTransition(int index, State state) {
 
     // TODO: we could interject that all trees must end with ROOT, for example
     if (state.tokenPosition >= state.sentence.size() && state.stack.size() == 1) {
-      return new OracleTransition(new FinalizeTransition(), false, false, false);
+      return new OracleTransition(new FinalizeTransition(rootStates), false, false, false);
     }
 
     if (state.stack.size() == 1) {