Permalink
Browse files

Refactored tests

  • Loading branch information...
1 parent a1a24aa commit b338545a2d5109130509353c69fe06aca7a7af63 @vpekar committed Feb 23, 2013
Showing with 14 additions and 13 deletions.
  1. +6 −4 stanford.py
  2. +8 −9 test_stanford.py
View
@@ -118,11 +118,11 @@ def get_lemma(self, word, tag):
return lemma.decode('latin1')
def get_pos_tag(self, node):
- parent = node_i.parent()
+ parent = node.parent()
tag = 'Z' if parent == None else parent.value()
return tag.decode('latin1')
- def get_dependency_data(self, node_i, idx):
+ def get_dependency_data(self, word, node_i, idx):
parent = self.gs.getGovernor(node_i)
if word in string.punctuation or parent == None:
parent_idx = 0
@@ -157,7 +157,7 @@ def populate_indices(self):
idx = node_i.index()
word = self.get_word(node_i)
tag = self.get_pos_tag(node_i)
- p_idx, rel = self.get_dependency_data(node_i, idx)
+ p_idx, rel = self.get_dependency_data(word, node_i, idx)
self.node[idx] = node_i
self.word[idx] = word
@@ -367,8 +367,10 @@ def parse_xml(self, text):
# build a plain-text token list and remember tag positions
xml_tags = {}
sent = []
+
for token in self.tokenize(text):
- token = unicode(token)
+ token = unicode(token).replace(u'\xa0', ' ')
+
if token.startswith('<'):
cur_size = len(sent)
xml_tags[cur_size] = xml_tags.get(cur_size, [])
View
@@ -5,12 +5,14 @@
from stanford import StanfordParser, PySentence
+PARSER = StanfordParser('englishPCFG.ser.gz')
+
+
class TestPySentence(unittest.TestCase):
def setUp(self):
input = "The quick brown fox jumped over a lazy dog."
- sp = StanfordParser('englishPCFG.ser.gz')
- self.sentence = sp.parse_xml(input)
+ self.sentence = PARSER.parse_xml(input)
def test_get_least_common_node(self):
lcn, shortest_path = self.sentence.get_least_common_node(4, 9)
@@ -26,13 +28,10 @@ def test_get_least_common_node(self):
class TestStanfordParser(unittest.TestCase):
- def setUp(self):
- self.sp = StanfordParser('englishPCFG.ser.gz')
-
def test_get_most_probable_parses_check_types(self):
input = 'I saw a man with a telescope.'
expected_type1, expected_type2 = type(PySentence), float
- for s, prob in self.sp.get_most_probable_parses(input, kbest=2):
+ for s, prob in PARSER.get_most_probable_parses(input, kbest=2):
actual_type1, actual_type2 = type(s.__class__), type(prob)
msg = "Expected %s != actual %s" % (expected_type1, actual_type1)
self.assertTrue(actual_type1 == expected_type1, msg)
@@ -42,23 +41,23 @@ def test_get_most_probable_parses_check_types(self):
def test_get_most_probable_parses_check_nonzero(self):
input = 'I saw a man with a telescope.'
expected = 2
- parses = [x for x in self.sp.get_most_probable_parses(input, kbest=expected)]
+ parses = [x for x in PARSER.get_most_probable_parses(input, kbest=expected)]
actual = len(parses)
msg = "Expected %d != actual %d" % (expected, actual)
self.assertTrue(expected == actual, msg)
def test_parse_xml(self):
input = 'This <a>is</a> a test<!-- b -->.'
expected = ['DT', 'VBZ', 'DT', 'NN', '.']
- sentence = self.sp.parse_xml(input)
+ sentence = PARSER.parse_xml(input)
actual = [v for k, v in sorted(sentence.tag.items())]
msg = "Expected %s != actual %s" % (expected, actual)
self.assertTrue(expected == actual, msg)
def test_tokenise(self):
input = 'This is a test.'
expected = ['This', 'is', 'a', 'test', '.']
- actual = [unicode(x) for x in self.sp.tokenize(input)]
+ actual = [unicode(x) for x in PARSER.tokenize(input)]
msg = "Expected %s != actual %s" % (expected, actual)
self.assertTrue(expected == actual, msg)

0 comments on commit b338545

Please sign in to comment.