-
Notifications
You must be signed in to change notification settings - Fork 0
/
utils.py
253 lines (212 loc) · 8.41 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
import math
from lxml import etree
from scipy.spatial import distance
import scipy
import re
import fnmatch
import numpy as np
import pandas as pd
import os
from numpy import dot
from numpy.linalg import norm
import sparse_coding as sc
import evaluation as eval
from evaluation import Level
DOCS = dict()
DATA_PATH = "data/Single/Source/DUC/"
SUMMARY_PATH = "data/Single/Summ/Extractive/"
def __content_processing(content):
content = content.replace("\n", " ").replace("(", " ").replace(")", " "). \
replace(" ", " ").replace(" ", " ")
sentences = re.split("\.|\?|\!", content)
while sentences.__contains__(''):
sentences.remove('')
while sentences.__contains__(' \n'):
sentences.remove(' \n')
for sentence in sentences:
words = sentence.split(" ")
while words.__contains__(''):
words.remove('')
if len(words) < 2:
sentences.remove(sentence)
words = list(set(map(lambda x: x.strip(), content.replace("?", " ").replace("!", " ").replace(".", " ").
replace("؟", " ").replace("!", " ").replace("،", " ").split(" "))))
if words.__contains__(''):
words.remove('')
return sentences, words
def read_document(doc_name):
file_path = DATA_PATH + doc_name
with open(file_path) as fp:
doc = fp.readlines()
content = ""
for line in doc:
content += line
fp.close()
return __content_processing(content)
def read_documents(directory_path):
# directory_path = "data/Multi/Track1/Source/D91A01/"
directory = os.fsencode(directory_path)
contents = ""
for file in os.listdir(directory):
filename = os.fsdecode(file)
content = etree.parse(directory_path + filename)
memoryElem = content.find('TEXT')
DOCS[filename] = memoryElem.text
contents += memoryElem.text
return __content_processing(contents)
def make_term_frequency(sentences, words):
term_frequency = dict()
for sentence in sentences:
vector = list()
for i in range(0, len(words)):
word = words[i]
vector.append(sentence.count(word))
if norm(vector) != 0:
term_frequency[sentence] = vector
# term_frequency[sentence] = vector / norm(vector, ord=1)
return term_frequency
def __avg_sent_2_vec(words, model):
M = []
for w in words:
try:
M.append(model[w])
except:
continue
M = np.array(M)
v = M.mean(axis=0)
return v / np.sqrt((v ** 2).sum())
def read_word2vec_model():
w2v = dict()
print("waiting to load word2vec model...")
with open('twitt_wiki_ham_blog.fa.text.100.vec', 'r', encoding='utf-8') as infile:
first_line = True
for line in infile:
if first_line:
first_line = False
continue
tokens = line.split()
w2v[tokens[0]] = [float(el) for el in tokens[1:]]
if len(w2v[tokens[0]]) != 100:
print('Bad line!')
print("model loaded")
return w2v
def make_word_2_vec(data, model):
word2vec = dict() # final dictionary containing sentence as the key and its representation as value
DocMatix = np.zeros((len(data), 100))
for i in range(len(data)):
words = list(map(lambda x: x.strip(), data[i].replace("?", " ").replace("!", " ").replace(".", " ").
replace("؟", " ").replace("!", " ").replace("،", " ").split(" ")))
if words.__contains__(''):
words.remove('')
result = __avg_sent_2_vec(words, model)
if not (np.isnan(result).any()):
DocMatix[i] = result
word2vec[data[i]] = DocMatix[i]
print("features calculated")
# print(word2vec)
train_df = pd.DataFrame(DocMatix)
# train_df.to_csv('AvgSent2vec.csv', index=False)
return word2vec
def __summary_vector_to_text_as_list_cosine(summary_set, representation):
summary_text = list()
for sen_vec in summary_set:
min_dist = 9999999 # max
min_sentence = ''
if norm(sen_vec) == 0:
print("sen vec zero")
for sentence in representation.keys():
if norm(representation[sentence]) == 0:
print("rep vec zero")
temp_dist = dot(sen_vec, representation[sentence]) / (norm(sen_vec) * norm(representation[sentence]))
if temp_dist < min_dist:
min_dist = temp_dist
min_sentence = sentence
summary_text.append(min_sentence)
return summary_text
def __summary_vector_to_text_as_list_euclidean(summary_set, representation):
summary_text = list()
for sen_vec in summary_set:
min_dist = 9999999 # max
min_sentence = ''
for sentence in representation.keys():
temp_dist = scipy.spatial.distance.euclidean(sen_vec, representation[sentence])
if temp_dist < min_dist:
min_dist = temp_dist
min_sentence = sentence
summary_text.append(min_sentence)
return summary_text
def __find_in_subdirectory(filename, subdirectory="data/Multi"):
if subdirectory:
path = subdirectory
else:
path = os.getcwd()
for root, dirs, names in os.walk(path):
if filename in names:
return os.path.join(root, filename)
return 'File not found'
def read_multi_ref_summaries(dir):
summaries = []
rootdir1 = "data/Multi"
summary_len = 0
for i in range(1, 9):
subDir = os.path.join(rootdir1, "Track" + str(i) + "/Summ/")
for root, dirs, files in os.walk(subDir):
for file in files:
if dir + '.E' in file:
with open(__find_in_subdirectory(file)) as fp:
doc = fp.readlines()
content = ""
for line in doc:
content += line
fp.close()
sentences = re.split("\.|\?|\!", content)
summary_len += len(sentences)
summaries.append(sentences)
return summaries, math.ceil(summary_len/5)
def read_single_ref_summaries(filename):
directory = os.fsencode(SUMMARY_PATH)
summaries = list()
summary_len = 0
for file in os.listdir(directory):
name = os.fsdecode(file)
if fnmatch.fnmatch(name, filename + '*'):
with open(SUMMARY_PATH + name) as summ_file:
lines = summ_file.readlines()
content = ''
for line in lines:
content += line
sentences = re.split("\.|\?|\!", content)
summary_len += len(sentences)
summaries.append(sentences)
summ_file.close()
return summaries, math.ceil(summary_len / 5)
def evaluate(representation, K, LAMBDA, t_stop, max_conse_rej, reference_summaries):
candidate_set = np.array(list([*v] for k, v in representation.items()))
summary_set = sc.MDS_sparse(candidate_set, K, LAMBDA, t_stop, max_conse_rej)
summary_text = __summary_vector_to_text_as_list_euclidean(summary_set, representation)
rouge_1_fscores = 0
rouge_2_fscores = 0
rouge_1_precisions = 0
rouge_2_precisions = 0
rouge_1_recalls = 0
rouge_2_recalls = 0
summ_len = 0
for summary_ref in reference_summaries:
summ_len += len(summary_ref)
rouge_1_fscore = eval.rouge_Fscore(summary_text, summary_ref, Level.Rouge_1)
rouge_1_fscores += rouge_1_fscore
rouge_1_precision = eval.rouge_precision(summary_text, summary_ref, Level.Rouge_1)
rouge_1_precisions += rouge_1_precision
rouge_1_recall = eval.rouge_recall(summary_text, summary_ref, Level.Rouge_1)
rouge_1_recalls += rouge_1_recall
rouge_2_fscore = eval.rouge_Fscore(summary_text, summary_ref, Level.Rouge_2)
rouge_2_fscores += rouge_2_fscore
rouge_2_precision = eval.rouge_precision(summary_text, summary_ref, Level.Rouge_2)
rouge_2_precisions += rouge_2_precision
rouge_2_recall = eval.rouge_recall(summary_text, summary_ref, Level.Rouge_2)
rouge_2_recalls += rouge_2_recall
print("Rouge-1 Fscore : ", rouge_1_fscores / 5)
print("Rouge-2 Fscore : ", rouge_2_fscores / 5)
print("------------------------------------")
return rouge_1_fscores / 5, rouge_2_fscores / 5, rouge_1_precisions / 5, rouge_2_precisions / 5, \
rouge_1_recalls / 5, rouge_2_recalls / 5