-
Notifications
You must be signed in to change notification settings - Fork 333
/
Postprocesser.py
84 lines (71 loc) · 2.37 KB
/
Postprocesser.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
from ..base.Dat import Dat, DATMaker
from ..base.compatibility import decodeGenerator
decode = decodeGenerator()
class Postprocesser():
def __init__(self, filename, tag, isTxt):
if(not filename):
return None
self.tag = tag
if(isTxt):
lexicon = []
f = open(filename, "r")
for i, line in enumerate(f):
line = line.split()
lexicon.append([decode(line[0]), i])
f.close()
dm = DATMaker()
dm.makeDat(lexicon, 0)
dm.shrink()
self.p_dat = Dat(datSize=dm.datSize, oldDat=dm.dat)
else:
self.p_dat = Dat(filename=filename)
def adjustSeg(self, sentence):
if(self.p_dat is None):
return
i = 0
while(i < len(sentence)):
tmp = sentence[i]
if(self.p_dat.getInfo(tmp) >= 0):
i += 1
continue
tmpVec = []
for j in range(i + 1, len(sentence)):
tmp += sentence[j]
if(self.p_dat.getInfo(tmp) >= 0):
break
tmpVec.append(tmp)
vecSize = len(tmpVec)
for k in range(vecSize-1, -1, -1):
tmp = tmpVec[k]
if(self.p_dat.match(tmp) != -1):
sentence[i] = tmp
del sentence[i+1:i+k+2]
break
i += 1
def adjustTag(self, sentence):
# print sentence
if(self.p_dat is None):
return
i = 0
while(i < len(sentence)):
tmp = sentence[i][0]
if(self.p_dat.getInfo(tmp) >= 0):
i+=1
continue
tmpVec = []
for j in range(i+1, len(sentence)):
tmp += sentence[j][0]
if(self.p_dat.getInfo(tmp) >= 0):
break
tmpVec.append(tmp)
vecSize = len(tmpVec)
for k in range(vecSize-1, -1, -1):
tmp = tmpVec[k]
if(self.p_dat.match(tmp) != -1):
sentence[i] = (tmp, '_', self.tag)
del sentence[i+1:i+k+2]
# sentence[i][2] = self.tag;
break
i+=1
if __name__ == '__main__':
pp = Postprocesser("userwords.txt", "uw", True)