Refactored tests

stanford.py

@@ -118,11 +118,11 @@ def get_lemma(self, word, tag):
         return lemma.decode('latin1')
 
     def get_pos_tag(self, node):
-        parent = node_i.parent()
+        parent = node.parent()
         tag = 'Z' if parent == None else parent.value()
         return tag.decode('latin1')
 
-    def get_dependency_data(self, node_i, idx):
+    def get_dependency_data(self, word, node_i, idx):
         parent = self.gs.getGovernor(node_i)
         if word in string.punctuation or parent == None:
             parent_idx = 0
@@ -157,7 +157,7 @@ def populate_indices(self):
             idx = node_i.index()
             word = self.get_word(node_i)
             tag = self.get_pos_tag(node_i)
-            p_idx, rel = self.get_dependency_data(node_i, idx)
+            p_idx, rel = self.get_dependency_data(word, node_i, idx)
 
             self.node[idx] = node_i
             self.word[idx] = word
@@ -367,8 +367,10 @@ def parse_xml(self, text):
         # build a plain-text token list and remember tag positions
         xml_tags = {}
         sent = []
+
         for token in self.tokenize(text):
-            token = unicode(token)
+            token = unicode(token).replace(u'\xa0', ' ')
+
             if token.startswith('<'):
                 cur_size = len(sent)
                 xml_tags[cur_size] = xml_tags.get(cur_size, [])

test_stanford.py

@@ -5,12 +5,14 @@
 from stanford import StanfordParser, PySentence
 
 
+PARSER = StanfordParser('englishPCFG.ser.gz')
+
+
 class TestPySentence(unittest.TestCase):
 
     def setUp(self):
         input = "The quick brown fox jumped over a lazy dog."
-        sp = StanfordParser('englishPCFG.ser.gz')
-        self.sentence = sp.parse_xml(input)
+        self.sentence = PARSER.parse_xml(input)
 
     def test_get_least_common_node(self):
         lcn, shortest_path = self.sentence.get_least_common_node(4, 9)
@@ -26,13 +28,10 @@ def test_get_least_common_node(self):
 
 class TestStanfordParser(unittest.TestCase):
 
-    def setUp(self):
-        self.sp = StanfordParser('englishPCFG.ser.gz')
-
     def test_get_most_probable_parses_check_types(self):
         input = 'I saw a man with a telescope.'
         expected_type1, expected_type2 = type(PySentence), float
-        for s, prob in self.sp.get_most_probable_parses(input, kbest=2):
+        for s, prob in PARSER.get_most_probable_parses(input, kbest=2):
             actual_type1, actual_type2 = type(s.__class__), type(prob)
             msg = "Expected %s != actual %s" % (expected_type1, actual_type1)
             self.assertTrue(actual_type1 == expected_type1, msg)
@@ -42,23 +41,23 @@ def test_get_most_probable_parses_check_types(self):
     def test_get_most_probable_parses_check_nonzero(self):
         input = 'I saw a man with a telescope.'
         expected = 2
-        parses = [x for x in self.sp.get_most_probable_parses(input, kbest=expected)]
+        parses = [x for x in PARSER.get_most_probable_parses(input, kbest=expected)]
         actual = len(parses)
         msg = "Expected %d != actual %d" % (expected, actual)
         self.assertTrue(expected == actual, msg)
 
     def test_parse_xml(self):
         input = 'This <a>is</a> a test<!-- b -->.'
         expected = ['DT', 'VBZ', 'DT', 'NN', '.']
-        sentence = self.sp.parse_xml(input)
+        sentence = PARSER.parse_xml(input)
         actual = [v for k, v in sorted(sentence.tag.items())]
         msg = "Expected %s != actual %s" % (expected, actual)
         self.assertTrue(expected == actual, msg)
 
     def test_tokenise(self):
         input = 'This is a test.'
         expected = ['This', 'is', 'a', 'test', '.']
-        actual = [unicode(x) for x in self.sp.tokenize(input)]
+        actual = [unicode(x) for x in PARSER.tokenize(input)]
         msg = "Expected %s != actual %s" % (expected, actual)
         self.assertTrue(expected == actual, msg)