Permalink
Browse files

Finished.

  • Loading branch information...
kerem kerem
kerem authored and kerem committed Jan 20, 2012
1 parent 9fbce6e commit 5dbed76279c2c6a22d8e23b9d8ed3533ad23de15
@@ -20,4 +20,12 @@ Brill accuracy: 0.794233 All
Brill accuracy: 0.794233 Prefixes only
Brill accuracy: 0.794110 Suffixes only
Brill accuracy: 0.794110 None
-Unigram: 0.792766
+Unigram: 0.792766
+
+2-pass Bigger set (MINOR_POS + VERB_INFLECTION + VERB_POLARITY)
+
+Brill accuracy: 0.758065
+
+2-pass Smaller set
+
+Brill accuracy: 0.741813
@@ -148,7 +148,7 @@ def extract_property(token, p1_token=None):
return token[0]
class ProximateP1Rule(ProximateWordsRule):
- PROPERTY_NAME = "suffix"
+ PROPERTY_NAME = "p1 tag"
def __init__(self, original_tag, replacement_tag, *conditions):
super(ProximateP1Rule, self).__init__(original_tag, replacement_tag, *conditions)
@@ -89,9 +89,9 @@ def write_errors_and_rules(error_output, rule_output, gold_data, testing_data, e
error_file.close()
print "Done; affix_rules and errors saved to %s and %s." % (rule_output, error_output)
-def demo2(num_sents=5635, max_rules=400, min_score=4,
+def demo2(num_sents=5635, max_rules=300, min_score=4,
error_output="errors.out", rule_output="rules.yaml",
- randomize=False, train=.85, trace=3):
+ randomize=False, train=.85, trace=4):
"""
Brill Tagger Demonstration
@@ -170,9 +170,10 @@ def demo2(num_sents=5635, max_rules=400, min_score=4,
for s, e in ((1, 1), (2, 2), (1, 2), (1, 3), (2, 3), (3, 3)):
templates.append(SymmetricProximateTokensTemplate(rule, (s, e)))
affix_rules = [Proximate1SuffixesRule, Proximate2SuffixesRule, Proximate3SuffixesRule
- , Proximate4SuffixesRule, Proximate5SuffixesRule, Proximate6SuffixesRule,
+ , Proximate4SuffixesRule, #Proximate5SuffixesRule, #Proximate6SuffixesRule,
Proximate1PrefixesRule, Proximate2PrefixesRule, Proximate3PrefixesRule
- , Proximate4PrefixesRule, Proximate5PrefixesRule, Proximate6PrefixesRule]
+ , Proximate4PrefixesRule]#, Proximate5PrefixesRule]
+ #, Proximate6PrefixesRule]
# affix_rules = [Proximate1PrefixesRule, Proximate2PrefixesRule, Proximate3PrefixesRule
# , Proximate4PrefixesRule, Proximate5PrefixesRule, Proximate6PrefixesRule]
# affix_rules = [Proximate1SuffixesRule, Proximate2SuffixesRule, Proximate3SuffixesRule
@@ -231,17 +232,20 @@ def demo2(num_sents=5635, max_rules=400, min_score=4,
# print tagger_pass
# exit()
templates = []
+# for rule in (ProximateTagsRule, ProximateWordsRule):
+# for s, e in ((1, 1), (2, 2), (1, 2), (1, 3), (2, 3), (3, 3)):
+# templates.append(SymmetricProximateTokensTemplate(rule, (s, e)))
+# affix_rules = [Proximate1SuffixesRule, Proximate2SuffixesRule, Proximate3SuffixesRule
+# , Proximate4SuffixesRule, #Proximate5SuffixesRule, #Proximate6SuffixesRule,
+# Proximate1PrefixesRule, Proximate2PrefixesRule, Proximate3PrefixesRule
+# , Proximate4PrefixesRule]
for rule in (ProximateTagsRule, ProximateWordsRule):
for s, e in ((1, 1), (2, 2), (1, 2), (1, 3), (2, 3), (3, 3)):
templates.append(SymmetricProximateTokensTemplate(rule, (s, e)))
- for rule in (Proximate1SuffixesRule, Proximate2SuffixesRule, Proximate3SuffixesRule
- , Proximate4SuffixesRule, Proximate5SuffixesRule, Proximate6SuffixesRule,
- Proximate7SuffixesRule, ProximateP1Rule):
+ for rule in affix_rules + [ProximateP1Rule]:
for s, e in zip([(-1, -1), (-2, -2)], [(1, 1), (2, 2)]):
templates.append(ProximateSuffixesTemplate(rule, s, e))
- for rule in (Proximate1SuffixesRule, Proximate2SuffixesRule, Proximate3SuffixesRule
- , Proximate4SuffixesRule, Proximate5SuffixesRule, Proximate6SuffixesRule,
- Proximate7SuffixesRule, ProximateP1Rule):
+ for rule in affix_rules + [ProximateP1Rule]:
for s, e in ((1, 1), (2, 2), (1, 2), (1, 3), (2, 3), (3, 3)):
templates.append(SymmetricProximateSuffixesTemplate(rule, (s, e)))
@@ -91,10 +91,10 @@ def _parse_word_tag(self, word):
MINOR = MINOR_POS
MAJOR = MAJOR_POS
if get_tagger_pass() == "P1":
- MINOR = MINOR_POS
+ MINOR = []
# MINOR = MINOR_POS + CASE + PERSON + VERB_INFLECTION + VERB_POLARITY
else:
- MINOR = MINOR_POS + VERB_INFLECTION
+ MINOR = MINOR_POS
for part in parts:
if part not in MAJOR:

0 comments on commit 5dbed76

Please sign in to comment.