Fix (Penn)TreePrintTest on Windows - use lineSeparator instead of \n

Author: asashour

build.xml

@@ -569,7 +569,7 @@
       <zipfileset prefix="WEB-INF/data/models"
                   file="/u/nlp/data/ner/goodClassifiers/english.muc.7class.distsim.crf.ser.gz"/>
       <zipfileset prefix="WEB-INF/data/models"
-                  file="/u/nlp/data/ner/goodClassifiers/german.distsim.crf.ser.gz"/>
+                  file="/u/nlp/data/ner/goodClassifiers/german.hgc_175m_600.crf.ser.gz"/>
       <zipfileset prefix="WEB-INF/data/models"
                   file="/u/nlp/data/ner/goodClassifiers/chinese.misc.distsim.crf.ser.gz"/>
       <zipfileset file="${data.path}/webapps/favicon.ico"/>

itest/src/edu/stanford/nlp/coref/CorefBenchmark.java

@@ -44,6 +44,7 @@ public void setUp() throws Exception, IOException {
     String[] corefArgs = { "-props", PROPERTIES_PATH};
     props = StringUtils.argsToProperties(corefArgs);
     props.setProperty("coref.conllOutputPath", WORK_DIR_FILE.getAbsolutePath()+"/");
+    props.setProperty("coref.verbose", "true");
 
     // build CorefSystem
     corefSystem = new CorefSystem(props);

itest/src/edu/stanford/nlp/ie/crf/ThreadedCRFClassifierITest.java

@@ -15,7 +15,7 @@ public class ThreadedCRFClassifierITest extends TestCase {
   Properties props;
 
   private static final String german1 =
-    "edu/stanford/nlp/models/ner/german.distsim.crf.ser.gz";
+    "edu/stanford/nlp/models/ner/german.conll.germeval2014.hgc_175m_600.crf.ser.gz";
   /** -- We're no longer supporting this one
   private String german2 =
     "/u/nlp/data/ner/goodClassifiers/german.dewac_175m_600.crf.ser.gz";

itest/src/edu/stanford/nlp/ie/crf/TrainCRFClassifierSlowITest.java

@@ -23,12 +23,12 @@ public void testGermanCRFClassifierTraining() throws Exception {
     StanfordRedwoodConfiguration.apply(PropertiesUtils.asProperties(
             "log.file", crfTrainingWorkingDir + "/german-crf.results"));
     // delete the model if present
-    File originalModelFile = new File(crfTrainingWorkingDir, "german.distsim.crf.ser.gz");
+    File originalModelFile = new File(crfTrainingWorkingDir, "german.hgc_175m_600.crf.ser.gz");
     originalModelFile.delete();
     // train the new model
     CRFClassifier.main(new String[] {
-            "-props", "edu/stanford/nlp/models/ner/german.distsim.prop",
-            "-serializeTo", crfTrainingWorkingDir+"/german.distsim.crf.ser.gz"
+            "-props", "edu/stanford/nlp/models/ner/german-2018.hgc_175m_600.prop",
+            "-serializeTo", crfTrainingWorkingDir+"/german.hgc_175m_600.crf.ser.gz"
     });
     List<String> germanTrainingResults = IOUtils.linesFromFile(crfTrainingWorkingDir + "/german-crf.results");
     String lastLineOfResults = germanTrainingResults.get(germanTrainingResults.size() - 1);

itest/src/edu/stanford/nlp/naturalli/OperatorScopeITest.java

@@ -4,7 +4,6 @@
 import edu.stanford.nlp.ling.CoreLabel;
 import edu.stanford.nlp.pipeline.Annotation;
 import edu.stanford.nlp.pipeline.StanfordCoreNLP;
-//import edu.stanford.nlp.trees.TreeCoreAnnotations;
 import edu.stanford.nlp.util.StringUtils;
 import org.junit.*;
 
@@ -39,7 +38,6 @@ public class OperatorScopeITest {
   private Optional<OperatorSpec>[] annotate(String text) {
     Annotation ann = new Annotation(text);
     pipeline.annotate(ann);
-    //System.out.println(ann.get(CoreAnnotations.SentencesAnnotation.class).get(0).get(TreeCoreAnnotations.TreeAnnotation.class));
     List<CoreLabel> tokens = ann.get(CoreAnnotations.SentencesAnnotation.class).get(0).get(CoreAnnotations.TokensAnnotation.class);
     Optional<OperatorSpec>[] scopes = new Optional[tokens.size()];
     Arrays.fill(scopes, Optional.empty());
@@ -322,7 +320,7 @@ public void fracasSentencesWithEvery() {
     checkScope("{ Every } [ customer who owns a computer ] [ has a service contract for it ]");
     checkScope("{ Every } [ department ] [ rents a line from BT ]");
     checkScope("{ Every } [ executive who had a laptop computer ] [ brought it to take notes at the meeting ]");
-    checkScope("{ Every } [ four - legged mammal ] [ is a four - legged animal ]");
+    checkScope("{ Every } [ four-legged mammal ] [ is a four-legged animal ]");
     checkScope("{ Every } [ individual who has the right to live anywhere in Europe ] [ can travel freely within Europe ]");
     checkScope("{ Every } [ individual who has the right to live in Europe ] [ can travel freely within Europe ]");
     checkScope("{ Every } [ inhabitant of Cambridge ] [ voted for a Labour MP ]");
@@ -420,21 +418,21 @@ public void fracasSentencesWithSome() {
   public void fracasSentencesWithThe() {
     checkScope("{ The } [ Ancient Greeks ] [ were all noted philosophers ]");
     checkScope("{ The } [ Ancient Greeks ] [ were noted philosophers ]");
-    checkScope("{ The } [ ITEL - XZ ] [ is fast ]");
-    checkScope("{ The } [ ITEL - ZX ] [ is an ITEL computer ]");
-    checkScope("{ The } [ ITEL - ZX ] [ is slower than 500 MIPS ]");
-    checkScope("{ The } [ PC - 6082 ] [ is as fast as the ITEL-XZ ]");
-    checkScope("{ The } [ PC - 6082 ] [ is fast ]");
-    checkScope("{ The } [ PC - 6082 ] [ is faster than 500 MIPS ]");
-    checkScope("{ The } [ PC - 6082 ] [ is faster than any ITEL computer ]");
-    checkScope("{ The } [ PC - 6082 ] [ is faster than every ITEL computer ]");
-    checkScope("{ The } [ PC - 6082 ] [ is faster than some ITEL computer ]");
-    checkScope("{ The } [ PC - 6082 ] [ is faster than the ITEL - XZ ]");
-    checkScope("{ The } [ PC - 6082 ] [ is faster than the ITEL - ZX ]");
-    checkScope("{ The } [ PC - 6082 ] [ is faster than the ITEL - ZX and the ITEL - ZY ]");
-    checkScope("{ The } [ PC - 6082 ] [ is faster than the ITEL - ZX or the ITEL - ZY ]");
-    checkScope("{ The } [ PC - 6082 ] [ is slow ]");
-    checkScope("{ The } [ PC - 6082 ] [ is slower than the ITEL - XZ ]");
+    checkScope("{ The } [ ITEL-XZ ] [ is fast ]");
+    checkScope("{ The } [ ITEL-ZX ] [ is an ITEL computer ]");
+    checkScope("{ The } [ ITEL-ZX ] [ is slower than 500 MIPS ]");
+    checkScope("{ The } [ PC-6082 ] [ is as fast as the ITEL-XZ ]");
+    checkScope("{ The } [ PC-6082 ] [ is fast ]");
+    checkScope("{ The } [ PC-6082 ] [ is faster than 500 MIPS ]");
+    checkScope("{ The } [ PC-6082 ] [ is faster than any ITEL computer ]");
+    checkScope("{ The } [ PC-6082 ] [ is faster than every ITEL computer ]");
+    checkScope("{ The } [ PC-6082 ] [ is faster than some ITEL computer ]");
+    checkScope("{ The } [ PC-6082 ] [ is faster than the ITEL-XZ ]");
+    checkScope("{ The } [ PC-6082 ] [ is faster than the ITEL-ZX ]");
+    checkScope("{ The } [ PC-6082 ] [ is faster than the ITEL-ZX and the ITEL-ZY ]");
+    checkScope("{ The } [ PC-6082 ] [ is faster than the ITEL-ZX or the ITEL-ZY ]");
+    checkScope("{ The } [ PC-6082 ] [ is slow ]");
+    checkScope("{ The } [ PC-6082 ] [ is slower than the ITEL-XZ ]");
     checkScope("{ The } [ chairman of the department ] [ is a person ]");
     checkScope("{ The } [ chairman ] [ read out every item on the agenda ]");
     checkScope("{ The } [ chairman ] [ read out the items on the agenda ]");
@@ -492,8 +490,7 @@ public void fracasSentencesWithProperNouns() {
     checkScope("[ { APCOM } ] [ has been paying mortgage interest for a total of 15 years or more ]");
     checkScope("[ { APCOM } ] [ lost some orders ]");
     checkScope("[ { APCOM } ] [ lost ten orders ]");
-    checkScope("[ { APCOM } ] [ signed the contract ]");
-    checkScope("[ { APCOM } ] [ signed the contract ] Friday");
+    checkScope("[ { APCOM } ] [ signed the contract Friday , 13th ]");
     checkScope("[ { APCOM } ] [ sold exactly 2500 computers ]");
     checkScope("[ { APCOM } ] [ won some orders ]");
     checkScope("[ { APCOM } ] [ won ten orders ]");
@@ -525,11 +522,11 @@ public void fracasSentencesWithProperNouns() {
     checkScope("[ { Bill } ] [ will speak to Mary ]");
     checkScope("[ { Bill } ] [ wrote a report ]");
 
-    checkScope("[ { Dumbo } ] [ is a four - legged animal ]");
+    checkScope("[ { Dumbo } ] [ is a four-legged animal ]");
     checkScope("[ { Dumbo } ] [ is a large animal ]");
     checkScope("[ { Dumbo } ] [ is a small animal ]");
     checkScope("[ { Dumbo } ] [ is a small elephant ]");
-    checkScope("[ { Dumbo } ] [ is four - legged ]");
+    checkScope("[ { Dumbo } ] [ is four-legged ]");
     checkScope("[ { Dumbo } ] [ is larger than Mickey ]");
 
     checkScope("[ { GFI } ] [ owns several computers ]");
@@ -550,7 +547,7 @@ public void fracasSentencesWithProperNouns() {
     checkScope("[ { ITEL } ] [ has developed a new editor since 1992 ]");
     checkScope("[ { ITEL } ] [ has expanded since 1992 ]");
     checkScope("[ { ITEL } ] [ has made a loss since 1992 ]");
-    checkScope("[ { ITEL } ] [ has sent most of the reports which Smith needs ]");
+    checkScope("[ { ITEL } ] [ has sent most of the reports Smith needs ]");
     checkScope("[ { ITEL } ] [ made a loss in 1993 ]");
     checkScope("[ { ITEL } ] [ maintains all the computers that GFI owns ]");
     checkScope("[ { ITEL } ] [ maintains them ]");
@@ -607,10 +604,8 @@ public void fracasSentencesWithProperNouns() {
     checkScope("[ { John } ] [ said Bill had been hurt ]");
     checkScope("[ { John } ] [ said Bill had hurt himself ]");
     checkScope("[ { John } ] [ said Bill wrote a report ]");
-    // FIXME this should work even if the parse changed some, right?
-    //checkScope("[ { John } ] [ said Mary wrote a report , and Bill did too ]");  // interesting example
-    // TODO(gabor) fix me (bad scope)
-    //checkScope("[ { John } ] [ said that Mary wrote a report ] , and that Bill did too");
+    checkScope("[ { John } ] [ said Mary wrote a report ] , and Bill did too");  // interesting example
+//    checkScope("[ { John } ] [ said that Mary wrote a report ] , and that Bill did too");  // TODO(gabor) fix me (bad scope)
     checkScope("[ { John } ] [ spoke to Mary ]");
     checkScope("[ { John } ] [ spoke to Mary at four o'clock ]");
     checkScope("[ { John } ] [ spoke to Mary on Friday ]");
@@ -623,11 +618,9 @@ public void fracasSentencesWithProperNouns() {
     checkScope("[ { John } ] [ wants to know how many women work part time ]");
     checkScope("[ { John } ] [ wants to know which men work part time ]");
     checkScope("[ { John } ] [ went to Paris by car ]");
-    // FIXME should this encompass "and Bill by train"?
-    // checkScope("[ { John } ] [ went to Paris by car , and Bill by train ]");
+    checkScope("[ { John } ] [ went to Paris by car , and Bill by train ]");
     checkScope("[ { John } ] [ went to Paris by car , and Bill by train to Berlin ]");
-    // FIXME should this encompass "and Bill to Berlin"?
-    // checkScope("[ { John } ] [ went to Paris by car , and Bill to Berlin ]");
+    checkScope("[ { John } ] [ went to Paris by car , and Bill to Berlin ]");
     checkScope("[ { John } ] [ wrote a report ]");
 //    checkScope("[ { John } ] [ wrote a report ] , and Bill said Peter did too ]");  // TODO(gabor) fix me
 

itest/src/edu/stanford/nlp/parser/nndep/CoNLLReadingITest.java

@@ -19,7 +19,7 @@
 
 public class CoNLLReadingITest extends TestCase {
 
-    String exampleCoNLLXPath = "/u/nlp/data/stanford-corenlp-testing/data/conllu/fr_gsd-ud-train.conllu.clean";
+    String exampleCoNLLXPath = "/u/nlp/data/stanford-corenlp/test/data/conllu/fr_gsd-ud-train.conllu.clean";
     String exampleCoNLLUPath = "";
 
     public static void loadConllFileOriginal(String inFile, List<CoreMap> sents, List<DependencyTree> trees, boolean unlabeled, boolean cPOS)

itest/src/edu/stanford/nlp/parser/shiftreduce/ShiftReduceParserITest.java

@@ -61,14 +61,14 @@ public void testBasicConstraint() {
     pq.setConstraints(constraints);
     assertTrue(pq.parse(sentence));
     result = pq.getBestParse();
-    assertEquals("(ROOT (S (NP (PRP It)) (VP (VBD was) (ADJP (NP (NP (NNP Carolina) (NNPS Reapers))))) (. .)))", result.toString());
+    assertEquals("(ROOT (S (NP (PRP It)) (VP (VBD was) (ADJP (NP (NNP Carolina) (NNPS Reapers)))) (. .)))", result.toString());
 
     constraint = new ParserConstraint(1, 3, "VP");
     constraints = Collections.singletonList(constraint);
     pq = englishParser.parserQuery();
     pq.setConstraints(constraints);
     assertTrue(pq.parse(sentence));
     result = pq.getBestParse();
-    assertEquals("(ROOT (S (NP (PRP It)) (VP (VBD was) (NP (NNP Carolina))) (NP (NNPS Reapers)) (. .)))", result.toString());
+    assertEquals("(ROOT (S (NP (PRP It)) (VP (VBD was) (NP (NNP Carolina))) (NNPS Reapers) (. .)))", result.toString()); 
   }
 }

itest/src/edu/stanford/nlp/pipeline/ChineseSegmenterRegressionITest.java

@@ -13,10 +13,10 @@ public class ChineseSegmenterRegressionITest  extends TestCase {
   public StanfordCoreNLP pipeline;
 
   // strings to test on
-  public List<String> inputStrings = new ArrayList<>();
+  public ArrayList<String> inputStrings = new ArrayList<>();
 
   // expected token lists
-  List<List<String>> expectedTokenLists = new ArrayList<>();
+  ArrayList<ArrayList<String>> expectedTokenLists = new ArrayList<>();
 
   public void setUp() {
 
@@ -31,33 +31,36 @@ public void setUp() {
 
     // example 1
     inputStrings.add("巴拉克·奥巴马是美国总统。他在2008年当选");
-    List<String> exampleOneTokenList = 
-        Arrays.asList("巴拉克·奥巴马","是","美国","总统","。","他","在","2008年","当选");
+    ArrayList exampleOneTokenList = new ArrayList<>(
+            Arrays.asList(
+                    new String[]{"巴拉克·奥巴马","是","美国","总统","。","他","在","2008年","当选"}));
     expectedTokenLists.add(exampleOneTokenList);
 
     // example 2
     inputStrings.add("声明全文如下:\n" +
-        "    \n" +
-        "   \n" +
-        "中国政府欢迎乌克兰销毁其境内全部核武器的决定,\n" +
-        "对乌克兰议会于11月16日批准乌克兰作为无核武器国\n" +
-        "家加入《不扩散核武器条约》表示赞赏。");
-    List<String> exampleTwoTokenList =
-        Arrays.asList("声明","全","文","如下",":","中国","政府","欢迎","乌克兰","销毁","其","境内",
-            "全部","核武器","的","决定",",","对","乌克兰","议会","于","11月","16日","批准","乌克兰","作为",
-            "无核武器","国家","加入","《","不扩散","核武器","条约","》","表示","赞赏","。");
+            "    \n" +
+            "   \n" +
+            "中国政府欢迎乌克兰销毁其境内全部核武器的决定,\n" +
+            "对乌克兰议会于11月16日批准乌克兰作为无核武器国\n" +
+            "家加入《不扩散核武器条约》表示赞赏。");
+    ArrayList exampleTwoTokenList =
+            new ArrayList<>(
+                    Arrays.asList(new String[]{"声明","全","文","如下",":","中国","政府","欢迎","乌克兰","销毁","其","境内",
+                            "全部","核武器","的","决定",",","对","乌克兰","议会","于","11月","16日","批准","乌克兰","作为",
+                            "无核武器","国家","加入","《","不扩散","核武器","条约","》","表示","赞赏","。"}));
     expectedTokenLists.add(exampleTwoTokenList);
 
     // example 3
     inputStrings.add("协定规定,自协定签署之日起一年后,缔约四国之间\n" +
-        "实现澜沧江-湄公河商船通航,缔约任何一方的船舶均可\n" +
-        "按照协定的规定在中国的思茅港和老挝的琅勃拉邦港之间\n" +
-        "自由航行。");
-    List<String> exampleThreeTokenList =
-        Arrays.asList("协定","规定",",","自","协定","签署","之日起","一","年","后",",","缔约","四",
-            "国","之间","实现","澜沧江","-","湄公河","商船","通航",",","缔约","任何","一","方","的","船舶",
-            "均","可","按照","协定","的","规定","在","中国","的","思茅","港","和","老挝","的","琅勃拉邦","港",
-            "之间","自由","航行","。");
+            "实现澜沧江-湄公河商船通航,缔约任何一方的船舶均可\n" +
+            "按照协定的规定在中国的思茅港和老挝的琅勃拉邦港之间\n" +
+            "自由航行。");
+    ArrayList exampleThreeTokenList =
+            new ArrayList<>(
+                    Arrays.asList(new String[]{"协定","规定",",","自","协定","签署","之日起","一","年","后",",","缔约","四",
+                            "国","之间","实现","澜沧江","-","湄公河","商船","通航",",","缔约","任何","一","方","的","船舶",
+                            "均","可","按照","协定","的","规定","在","中国","的","思茅","港","和","老挝","的","琅勃拉邦","港",
+                            "之间","自由","航行","。"}));
     expectedTokenLists.add(exampleThreeTokenList);
   }
 

itest/src/edu/stanford/nlp/pipeline/NERTokenizationITest.java

@@ -47,7 +47,7 @@ public void testEnglishNERTokenization() {
     List<String> docs = new ArrayList<>();
     List<List<String>> words = new ArrayList<>();
     List<List<String>> labels = new ArrayList<>();
-    List<List<CoreLabel>> goldTokens = new ArrayList<>();
+    List<List<CoreLabel>> goldTokens = new ArrayList();
     // basic example
     docs.add("Barack Obama was born in Hawaii.");
     words.add(Arrays.asList("Barack", "Obama", "was", "born", "in", "Hawaii", "."));
@@ -87,7 +87,7 @@ public void testEnglishNERTokenizationJustStatistical() {
     List<String> docs = new ArrayList<>();
     List<List<String>> words = new ArrayList<>();
     List<List<String>> labels = new ArrayList<>();
-    List<List<CoreLabel>> goldTokens = new ArrayList<>();
+    List<List<CoreLabel>> goldTokens = new ArrayList();
     // basic example
     docs.add("Barack Obama was born in Hawaii.");
     words.add(Arrays.asList("Barack", "Obama", "was", "born", "in", "Hawaii", "."));
@@ -128,7 +128,7 @@ public void testEnglishNERTokenizationWithPTB3EscapingJustStatistical() {
     List<String> docs = new ArrayList<>();
     List<List<String>> words = new ArrayList<>();
     List<List<String>> labels = new ArrayList<>();
-    List<List<CoreLabel>> goldTokens = new ArrayList<>();
+    List<List<CoreLabel>> goldTokens = new ArrayList();
     // basic example
     docs.add("Barack Obama was born in Hawaii.");
     words.add(Arrays.asList("Barack", "Obama", "was", "born", "in", "Hawaii", "."));
@@ -167,7 +167,7 @@ public void testEnglishNERTokenizationTurnedOff() {
     List<String> docs = new ArrayList<>();
     List<List<String>> words = new ArrayList<>();
     List<List<String>> labels = new ArrayList<>();
-    List<List<CoreLabel>> goldTokens = new ArrayList<>();
+    List<List<CoreLabel>> goldTokens = new ArrayList();
     // basic example with "-"
     docs.add("She traveled to Port-au-Prince over the summer with Jane Smith.");
     words.add(Arrays.asList("She", "traveled", "to", "Port", "-", "au", "-", "Prince", "over", "the", "summer", "with",
@@ -198,7 +198,7 @@ public void testGermanNERTokenization() {
     List<String> docs = new ArrayList<>();
     List<List<String>> words = new ArrayList<>();
     List<List<String>> labels = new ArrayList<>();
-    List<List<CoreLabel>> goldTokens = new ArrayList<>();
+    List<List<CoreLabel>> goldTokens = new ArrayList();
     // basic example with "-"
     docs.add("Die Microsoft-Aktie sank daraufhin an der Wall Street um über vier Dollar auf 89,87 Dollar.");
     words.add(Arrays.asList("Die", "Microsoft", "-", "Aktie", "sank", "daraufhin", "an", "der", "Wall", "Street",

itest/src/edu/stanford/nlp/pipeline/POSTaggerBenchmarkITest.java

@@ -35,13 +35,13 @@ public void testEnglishBiDirectionalWSJTestPOS() throws IOException {
 
   public void testEnglishCaselessWSJDevPOS() throws IOException {
     runPOSTest("edu/stanford/nlp/models/pos-tagger/english-caseless-left3words-distsim.tagger",
-            "tagSeparator=_,/u/nlp/data/pos-tagger/models-4.0.0/data/wsj-caseless/dev/wsj-caseless-dev.txt",
-            95.80);
+            "format=TSV,wordColumn=0,tagColumn=1,/u/nlp/data/pos-tagger/english/test-wsj-19-21.caseless.tsv",
+            95.90);
   }
 
   public void testEnglishCaselessWSJTestPOS() throws IOException {
     runPOSTest("edu/stanford/nlp/models/pos-tagger/english-caseless-left3words-distsim.tagger",
-            "tagSeparator=_,/u/nlp/data/pos-tagger/models-4.0.0/data/wsj-caseless/test/wsj-caseless-test.txt",
+            "format=TSV,wordColumn=0,tagColumn=1,/u/nlp/data/pos-tagger/english/test-wsj-22-24.caseless.tsv",
             95.92);
   }