-
Notifications
You must be signed in to change notification settings - Fork 2
/
errorAnalysis.py
155 lines (124 loc) · 7.36 KB
/
errorAnalysis.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
#!/usr/bin/python3
#author: Fan Luo
import numpy as np
import string
import math
import sys
import argparse
def create_parser():
parser = argparse.ArgumentParser(description='Arguments')
parser.add_argument('--predictionFileModal', type=str, help="")
parser.add_argument('--predictionFileNoModal', type=str, help="")
parser.add_argument('--outputFile', type=str, help="")
return parser
def parse_commandline_args():
return create_parser().parse_args()
# take the 'error' field for sort
def takeError(d):
return d['error']
def main():
if len(sys.argv) != 7:
print("usage: python errorAnalysis.py --predictionFileModal [INPUT_FILE] --predictionFileNoModal [INPUT_FILE] --outputFile [OUTPUT_FILE]")
else:
args = parse_commandline_args()
# Compare between before and after removing modal
modals = ["can", "could", "may", "might", "should", "would"]
# this is the dicts for before removing modal, each dict corresponding to one instance/line;
# the dicts also used for printing, so combined info from after removing modal
keys = ['modalPredication', 'label', 'predicatePosition', 'sentence', 'sentWithoutModal', 'modalPositions', 'NoModalPredication', 'error', 'NoModalError']
values = []
dicts = []
# this is for after removing modal
NoModalKeys = ['NoModalPredication','label','NoModalPredicatePosition','sentWithoutModal']
NoModalvalues = []
NoModalDicts = []
with open(args.predictionFileModal) as f1:
with open(args.predictionFileNoModal) as f2:
# store info of after removing modal into NoModalDicts
for line in f2:
line = line.strip().split()
if(len(line) > 3):
NoModalvalues = line[:3]
NoModalvalues.append(" ".join(line[3:])) # sentence without modal
d = dict(zip(NoModalKeys, NoModalvalues))
NoModalDicts.append(d)
# store info of before removing modal, also combine info from after removing modal for printing
for line in f1:
# Split each line.
line = line.strip().split()
if(len(line) > 3):
values = line[:3]
values.append(" ".join(line[3:])) # sentence
sentWithoutModal = []
modalPositions=[]
for i, w in enumerate(line[3:]):
if(w.lower() not in modals): # if the word is not a modal, append to sentWithoutModal
sentWithoutModal.append(w)
else: # a word is a modal, record its position, to be marked when printing
modalPositions.append(i)
values.append(sentWithoutModal)
values.append(modalPositions)
NoModalPredication = ''
if (len(modalPositions) == 0): # NoModalPredication will be same as ModalPredication, since there is no modal, sentence are same
NoModalPredication = line[0]
# This is for debug, if no modal in the sentence, the prediction of before and after should be same
# for NoModalDict in NoModalDicts:
# if (" ".join(line[3:]) in list(NoModalDict.values()) and NoModalDict['label']==line[1] and line[2]==NoModalDict['NoModalPredicatePosition']):
# print("match")
# if(NoModalPredication != NoModalDict['NoModalPredication']):
# print("error")
# print(line)
# print(NoModalPredication)
# print(NoModalDict['NoModalPredication'])
else: # at least one modal exists
for NoModalDict in NoModalDicts:
if (" ".join(sentWithoutModal) in list(NoModalDict.values()) and NoModalDict['label']==line[1]):
# print(NoModalDict['NoModalPredicatePosition'])
# print(line[2])
# print(len(modalPositions))
if(int(NoModalDict['NoModalPredicatePosition']) <= int(line[2]) and int(NoModalDict['NoModalPredicatePosition']) >= int(line[2]) - len(modalPositions)): # line[2]) is predicate position. if modal is after predicate, its position is not afffected though
NoModalPredication = NoModalDict['NoModalPredication']
if(NoModalPredication == ''):
print("Did not found a corresponding sentence after removing modal")
# print(line)
# print(sentWithoutModal)
values.append(NoModalPredication)
values.append(abs(float(line[0]) - float(line[1]))) # abs error: diff between prediction and label
values.append(abs(float(NoModalPredication) - float(line[1]))) # abs NoModalError
# Create dict for each row.
d = dict(zip(keys, values))
dicts.append(d)
#sort dicts according to error before removing modal
sortedDicts = sorted(dicts, reverse=True, key=takeError)
# print output
with open(args.outputFile, 'w') as output:
output.write("label\twithModalPrediction\tNoModalPrediction\twithModalDiff\tNoModalDiff\twithout modal\tsentence\n")
for sortedDict in sortedDicts:
output.write(sortedDict['label'])
output.write('\t')
output.write(sortedDict['modalPredication'])
output.write('\t')
output.write(sortedDict['NoModalPredication'])
output.write('\t')
output.write(str(sortedDict['error']))
output.write('\t')
output.write(str(sortedDict['NoModalError']))
output.write('\t')
if (len(sortedDict['modalPositions']) == 0):
output.write('-\t')
elif(float(sortedDict['NoModalError'])+0.001 < float(sortedDict['error'])):
output.write('better\t')
elif(float(sortedDict['NoModalError'])-0.001 > float(sortedDict['error'])):
output.write('worse\t')
else:
output.write('same\t')
for i, w in enumerate(sortedDict['sentence'].split()):
if(i == sortedDict['predicatePosition']):
output.write('**' + w + '**\t') # wrap the word by '**' to denote this is the predicate
elif(i in sortedDict['modalPositions']):
output.write('--' + w + '--\t') # wrap the word by '--' to denote this is the removed modal
else:
output.write( w + '\t')
output.write('\n')
if __name__ == '__main__':
main()