-
-
Notifications
You must be signed in to change notification settings - Fork 270
/
feature.py
91 lines (78 loc) · 2.41 KB
/
feature.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
import time
from underthesea.corpus.readers.dictionary_loader import DictionaryLoader
words = set(DictionaryLoader('Viet74K.txt').words)
def word2features(sent, i):
"""
add feature for each word
:param unicode|str sent: input sentence
:param int i: index of word in sentence
:return: word added feature
:rtype: list
"""
word = sent[i][0]
features = [
word,
'bias',
'word.lower=' + word.lower(),
'word.isupper=%s' % word.isupper(),
'word.istitle=%s' % word.istitle(),
'word.isdigit=%s' % word.isdigit(),
]
if i >= 1:
word1 = sent[i - 1][0]
features.extend([
'-1:word.lower=' + word1.lower(),
'-1:word.istitle=%s' % word1.istitle(),
'-1:word.isupper=%s' % word1.isupper(),
])
else:
features.append('BOS')
compare_dict_start = time.time()
if i < len(sent) - 1:
word = " ".join([sent[i][0].lower(), sent[i + 1][0].lower()])
features.extend([
'2_word.in_dictionary=' + str(int(word in words)),
])
else:
features.append('2_word:EOS')
if i >= 1:
word = " ".join([sent[i - 1][0].lower(), sent[i][0].lower()])
features.extend([
'-2_word.in_dictionary=' + str(int(word in words)),
])
else:
features.append('-2_word:BOS')
if i >= 2:
word = " ".join([sent[i - 2][0].lower(), sent[i - 1][0].lower(), sent[i][0].lower()])
features.extend([
'-3_word.in_dictionary=' + str(int(word in words)),
])
else:
features.append('-3_word:BOS')
if i < len(sent) - 2:
word = " ".join([sent[i][0].lower(), sent[i + 1][0].lower(), sent[i + 2][0]])
features.extend([
'3_word.in_dictionary=' + str(int(word in words)),
])
else:
features.append('3_word:EOS')
if i >= 2:
word2 = sent[i - 2][0]
features.extend(['-2:word.lower=' + word2.lower()])
else:
features.append('BOS')
if i < len(sent) - 1:
next_word_1 = sent[i + 1][0]
features.extend([
'+1:word.lower=' + next_word_1.lower()
])
else:
features.append('EOS')
if i < len(sent) - 2:
next_word_2 = sent[i + 2][0]
features.extend([
'+2:word.lower=' + next_word_2.lower()
])
else:
features.append('EOS')
return features