/
scores.py
97 lines (84 loc) · 3.53 KB
/
scores.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
#!/usr/bin/env python
#coding=utf-8
'''
Computes AMR scores for concept identification, named entity recognition, wikification,
negation detection, reentrancy detection and SRL.
@author: Marco Damonte (m.damonte@sms.ed.ac.uk)
@since: 03-10-16
'''
import sys
import smatch.amr as amr
import smatch.smatch_fromlists as smatch
from collections import defaultdict
from utils import *
pred = open(sys.argv[1]).read().strip().split("\n\n")
gold = open(sys.argv[2]).read().strip().split("\n\n")
inters = defaultdict(int)
golds = defaultdict(int)
preds = defaultdict(int)
reentrancies_pred = []
reentrancies_gold = []
srl_pred = []
srl_gold = []
k = 0
tot = 0
correct = 0
for amr_pred, amr_gold in zip(pred, gold):
amr_pred = amr.AMR.parse_AMR_line(amr_pred.replace("\n",""))
dict_pred = var2concept(amr_pred)
triples_pred = []
for t in amr_pred.get_triples()[1] + amr_pred.get_triples()[2]:
if t[0].endswith('-of'):
triples_pred.append((t[0][:-3], t[2], t[1]))
else:
triples_pred.append((t[0], t[1], t[2]))
amr_gold = amr.AMR.parse_AMR_line(amr_gold.replace("\n",""))
dict_gold = var2concept(amr_gold)
triples_gold = []
for t in amr_gold.get_triples()[1] + amr_gold.get_triples()[2]:
if t[0].endswith('-of'):
triples_gold.append((t[0][:-3], t[2], t[1]))
else:
triples_gold.append((t[0], t[1], t[2]))
list_pred = disambig(concepts(dict_pred))
list_gold = disambig(concepts(dict_gold))
inters["Concepts"] += len(list(set(list_pred) & set(list_gold)))
preds["Concepts"] += len(set(list_pred))
golds["Concepts"] += len(set(list_gold))
list_pred = disambig(namedent(dict_pred, triples_pred))
list_gold = disambig(namedent(dict_gold, triples_gold))
inters["Named Ent."] += len(list(set(list_pred) & set(list_gold)))
preds["Named Ent."] += len(set(list_pred))
golds["Named Ent."] += len(set(list_gold))
list_pred = disambig(negations(dict_pred, triples_pred))
list_gold = disambig(negations(dict_gold, triples_gold))
inters["Negations"] += len(list(set(list_pred) & set(list_gold)))
preds["Negations"] += len(set(list_pred))
golds["Negations"] += len(set(list_gold))
list_pred = disambig(wikification(triples_pred))
list_gold = disambig(wikification(triples_gold))
inters["Wikification"] += len(list(set(list_pred) & set(list_gold)))
preds["Wikification"] += len(set(list_pred))
golds["Wikification"] += len(set(list_gold))
reentrancies_pred.append(reentrancies(dict_pred, triples_pred))
reentrancies_gold.append(reentrancies(dict_gold, triples_gold))
srl_pred.append(srl(dict_pred, triples_pred))
srl_gold.append(srl(dict_gold, triples_gold))
for score in preds:
if preds[score] > 0:
pr = inters[score]/float(preds[score])
else:
pr = 0
if golds[score] > 0:
rc = inters[score]/float(golds[score])
else:
rc = 0
if pr + rc > 0:
f = 2*(pr*rc)/(pr+rc)
print (score, '-> P:', "{0:.2f}".format(pr), ', R:', "{0:.2f}".format(rc), ', F:', "{0:.2f}".format(f))
else:
print (score, '-> P:', "{0:.2f}".format(pr), ', R:', "{0:.2f}".format(rc), ', F: 0.00')
pr, rc, f = smatch.main(reentrancies_pred, reentrancies_gold, True)
print ('Reentrancies -> P:', "{0:.2f}".format(float(pr)), ', R:', "{0:.2f}".format(float(rc)), ', F:', "{0:.2f}".format(float(f)))
pr, rc, f = smatch.main(srl_pred, srl_gold, True)
print ('SRL -> P:', "{0:.2f}".format(float(pr)), ', R:', "{0:.2f}".format(float(rc)), ', F:', "{0:.2f}".format(float(f)))