Update NP extracting scripts

exp/alto/tools/get_np_terminals.py

@@ -1,4 +1,5 @@
 import sys
+
 from nltk.tree import Tree
 
 TEMPLATE = ('{0} -> {1}_{0}\n[tree] {0}({1})\n[ud] "({1}<root> / {1})"\n' +
@@ -16,7 +17,7 @@ def get_np_terminals():
 
     #print(seen)
     for tuples in seen:
-         print(TEMPLATE.format(tuples[1], tuples[0]))
+         print TEMPLATE.format(tuples[1], tuples[0])
 
 
-get_np_terminals()
+get_np_terminals()

exp/alto/tools/get_nps_from_treebanks.py → exp/alto/tools/get_nps_from_treebank.py

@@ -1,9 +1,20 @@
 from nltk.corpus import treebank
 
+from common import sanitize_word
+
+def format_tree(tree):
+    if tree.height() == 2: #word, pos
+        tree[0] = sanitize_word(tree[0]) #tree[0] == word
+    else:
+        for subtree in tree:
+            format_tree(subtree)
+
+
 for n in range(1,200): 
     tree_file = "wsj_{}.mrg".format(str(n).zfill(4))
     sentences = treebank.parsed_sents(tree_file)
     for s in sentences:
         for subtree in s.subtrees(lambda t: t.label() == "NP"):
+            format_tree(subtree)
             print(subtree.pformat(100000))
             #print(subtree)