Add a main program to NonDict2 which rebuilds the dictionary from a segmenter training file

Author: AngledLuffa

src/edu/stanford/nlp/ie/AbstractSequenceClassifier.java

@@ -252,14 +252,14 @@ public DocumentReaderAndWriter<IN> makeReaderAndWriter() {
    * edu.stanford.nlp.wordseg.Sighan2005DocumentReaderAndWriter for
    * the Chinese Segmenter.
    */
-  public DocumentReaderAndWriter<IN> makePlainTextReaderAndWriter() {
+  public static <INN extends CoreMap> DocumentReaderAndWriter<INN> makePlainTextReaderAndWriter(SeqClassifierFlags flags) {
     String readerClassName = flags.plainTextDocumentReaderAndWriter;
     // We set this default here if needed because there may be models
     // which don't have the reader flag set
     if (readerClassName == null) {
       readerClassName = SeqClassifierFlags.DEFAULT_PLAIN_TEXT_READER;
     }
-    DocumentReaderAndWriter<IN> readerAndWriter;
+    DocumentReaderAndWriter<INN> readerAndWriter;
     try {
       readerAndWriter = ReflectionLoading.loadByReflection(readerClassName);
     } catch (Exception e) {
@@ -269,6 +269,10 @@ public DocumentReaderAndWriter<IN> makePlainTextReaderAndWriter() {
     return readerAndWriter;
   }
 
+  public DocumentReaderAndWriter<IN> makePlainTextReaderAndWriter() {
+    return makePlainTextReaderAndWriter(flags);
+  }
+
   /**
    * Returns the background class for the classifier.
    *

src/edu/stanford/nlp/sequences/SeqClassifierFlags.java

@@ -1081,6 +1081,8 @@ public enum SlashHyphenEnum { NONE, WFRAG, WORD, BOTH };
    */
   public boolean useMoreNeighborNGrams = false;
 
+  /** if using dict2 in a segmenter, load it with this filename */
+  public String dict2name = "";
 
   // "ADD VARIABLES ABOVE HERE"
 
@@ -2634,6 +2636,8 @@ public void setProperties(Properties props, boolean printProps) {
       } else if (key.equalsIgnoreCase("ner.model")) {
         nerModel = val;
       } else if (key.equalsIgnoreCase("sutime.language")) {
+      } else if (key.equalsIgnoreCase("dict2name")) {
+        dict2name = val;
         // ADD VALUE ABOVE HERE
       } else if ( ! key.isEmpty() && ! key.equals("prop")) {
         log.info("Unknown property: |" + key + '|');

src/edu/stanford/nlp/wordseg/NonDict2.java

@@ -1,9 +1,21 @@
 package edu.stanford.nlp.wordseg;
 
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
 
-import edu.stanford.nlp.util.logging.Redwood;
-
+import edu.stanford.nlp.ie.AbstractSequenceClassifier;
+import edu.stanford.nlp.io.IOUtils;
+import edu.stanford.nlp.ling.CoreLabel;
+import edu.stanford.nlp.sequences.DocumentReaderAndWriter;
 import edu.stanford.nlp.sequences.SeqClassifierFlags;
+import edu.stanford.nlp.util.StringUtils;
+import edu.stanford.nlp.util.logging.Redwood;
 
 public class NonDict2  {
 
@@ -23,7 +35,10 @@ public NonDict2(SeqClassifierFlags flags) {
     }
 
     String path;
-    if (flags.useAs || flags.useHk || flags.useMsr) {
+    if (flags.dict2name != null && !flags.dict2name.equals("")) {
+      path = corporaDict + "/dict/" + flags.dict2name;
+      logger.info("INFO: dict2name specified | building NonDict2 from "+path);
+    } else if (flags.useAs || flags.useHk || flags.useMsr) {
       throw new RuntimeException("only support settings for CTB and PKU now.");
     } else if ( flags.usePk ) {
       path = corporaDict+"/dict/pku.non";
@@ -43,4 +58,65 @@ public String checkDic(String c2, SeqClassifierFlags flags) {
     return "0";
   }
 
+  /**
+   * Rebuilds a non-dict.  Use -textFile and -outputFile as appropriate.
+   * Uses SeqClassifierFlags so that specific flags for the reader can be honored.
+   */
+  public static void main(String[] args) throws IOException {
+    Properties props = StringUtils.argsToProperties(args, SeqClassifierFlags.flagsToNumArgs());
+
+    /*
+    // TODO: refactor this into a util?
+    // TODO: whitespace reader
+    boolean foundReader = false;
+    for (String propKey : props.stringPropertyNames()) {
+      if (propKey.equalsIgnoreCase("plainTextDocumentReaderAndWriter")) {
+        foundReader = true;
+        break;
+      }
+    }
+    if (!foundReader) {
+      // this doesn't exist
+      props.setProperty("plainTextDocumentReaderAndWriter", "edu.stanford.nlp.sequences.WhitespaceDocumentReaderAndWriter");
+    }
+    */
+
+    SeqClassifierFlags flags = new SeqClassifierFlags(props);
+
+    String inputFilename = flags.textFile;
+    String outputFilename = flags.outputFile;
+
+    DocumentReaderAndWriter<CoreLabel> readerAndWriter = AbstractSequenceClassifier.makePlainTextReaderAndWriter(flags);
+    readerAndWriter.init(flags);
+
+    Set<String> splitBigrams = new HashSet<>();
+
+    FileReader fin = new FileReader(inputFilename);
+    // for some weird syntax reason this can't take the place of ': iterable'
+    Iterable<List<CoreLabel>> iterable = () -> readerAndWriter.getIterator(fin);
+    List<CoreLabel> prevSentence = null;
+    for (List<CoreLabel> sentence : iterable) {
+      for (int i = 0; i < sentence.size() - 1; ++i) {
+        String prevWord = sentence.get(i).value();
+        String nextWord = sentence.get(i+1).value();
+        String bigram = prevWord.substring(prevWord.length() - 1) + nextWord.substring(0, 1);
+        splitBigrams.add(bigram);
+      }
+      if (prevSentence != null) {
+        String prevWord = prevSentence.get(prevSentence.size() - 1).value();
+        String nextWord = sentence.get(0).value();
+        String bigram = prevWord.substring(prevWord.length() - 1) + nextWord.substring(0, 1);
+        splitBigrams.add(bigram);
+      }
+      prevSentence = sentence;
+    }
+    fin.close();
+
+    PrintWriter fout = IOUtils.getPrintWriter(outputFilename, "utf-8");
+    for (String bigram : splitBigrams) {
+      fout.print(bigram);
+      fout.println();
+    }
+    fout.close();
+  }
 }