/
__init__.py
185 lines (141 loc) · 5.59 KB
/
__init__.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
"""Python implementation of the TextRank algoritm.
From this paper:
https://web.eecs.umich.edu/~mihalcea/papers/mihalcea.emnlp04.pdf
Based on:
https://gist.github.com/voidfiles/1646117
https://github.com/davidadamojr/TextRank
"""
import editdistance
import io
import itertools
import networkx as nx
import nltk
import os
def setup_environment():
"""Download required resources."""
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
print('Completed resource downloads.')
def filter_for_tags(tagged, tags=['NN', 'JJ', 'NNP']):
"""Apply syntactic filters based on POS tags."""
return [item for item in tagged if item[1] in tags]
def normalize(tagged):
"""Return a list of tuples with the first item's periods removed."""
return [(item[0].replace('.', ''), item[1]) for item in tagged]
def unique_everseen(iterable, key=None):
"""List unique elements in order of appearance.
Examples:
unique_everseen('AAAABBBCCDAABBB') --> A B C D
unique_everseen('ABBCcAD', str.lower) --> A B C D
"""
seen = set()
seen_add = seen.add
if key is None:
def key(x): return x
for element in iterable:
k = key(element)
if k not in seen:
seen_add(k)
yield element
def build_graph(nodes):
"""Return a networkx graph instance.
:param nodes: List of hashables that represent the nodes of a graph.
"""
gr = nx.Graph() # initialize an undirected graph
gr.add_nodes_from(nodes)
nodePairs = list(itertools.combinations(nodes, 2))
# add edges to the graph (weighted by Levenshtein distance)
for pair in nodePairs:
firstString = pair[0]
secondString = pair[1]
levDistance = editdistance.eval(firstString, secondString)
gr.add_edge(firstString, secondString, weight=levDistance)
return gr
def extract_key_phrases(text):
"""Return a set of key phrases.
:param text: A string.
"""
# tokenize the text using nltk
word_tokens = nltk.word_tokenize(text)
# assign POS tags to the words in the text
tagged = nltk.pos_tag(word_tokens)
textlist = [x[0] for x in tagged]
tagged = filter_for_tags(tagged)
tagged = normalize(tagged)
unique_word_set = unique_everseen([x[0] for x in tagged])
word_set_list = list(unique_word_set)
# this will be used to determine adjacent words in order to construct
# keyphrases with two words
graph = build_graph(word_set_list)
# pageRank - initial value of 1.0, error tolerance of 0,0001,
calculated_page_rank = nx.pagerank(graph, weight='weight')
# most important words in ascending order of importance
keyphrases = sorted(calculated_page_rank, key=calculated_page_rank.get,
reverse=True)
# the number of keyphrases returned will be relative to the size of the
# text (a third of the number of vertices)
one_third = len(word_set_list) // 3
keyphrases = keyphrases[0:one_third + 1]
# take keyphrases with multiple words into consideration as done in the
# paper - if two words are adjacent in the text and are selected as
# keywords, join them together
modified_key_phrases = set([])
i = 0
while i < len(textlist):
w = textlist[i]
if w in keyphrases:
phrase_ws = [w]
i += 1
while i < len(textlist) and textlist[i] in keyphrases:
phrase_ws.append(textlist[i])
i += 1
phrase = ' '.join(phrase_ws)
if phrase not in modified_key_phrases:
modified_key_phrases.add(phrase)
else:
i += 1
return modified_key_phrases
def extract_sentences(text, summary_length=100, clean_sentences=False, language='english'):
"""Return a paragraph formatted summary of the source text.
:param text: A string.
"""
sent_detector = nltk.data.load('tokenizers/punkt/'+language+'.pickle')
sentence_tokens = sent_detector.tokenize(text.strip())
graph = build_graph(sentence_tokens)
calculated_page_rank = nx.pagerank(graph, weight='weight')
# most important sentences in ascending order of importance
sentences = sorted(calculated_page_rank, key=calculated_page_rank.get,
reverse=True)
# return a 100 word summary
summary = ' '.join(sentences)
summary_words = summary.split()
summary_words = summary_words[0:summary_length]
dot_indices = [idx for idx, word in enumerate(summary_words) if word.find('.') != -1]
if clean_sentences and dot_indices:
last_dot = max(dot_indices) + 1
summary = ' '.join(summary_words[0:last_dot])
else:
summary = ' '.join(summary_words)
return summary
def write_files(summary, key_phrases, filename):
"""Write key phrases and summaries to a file."""
print("Generating output to " + 'keywords/' + filename)
key_phrase_file = io.open('keywords/' + filename, 'w')
for key_phrase in key_phrases:
key_phrase_file.write(key_phrase + '\n')
key_phrase_file.close()
print("Generating output to " + 'summaries/' + filename)
summary_file = io.open('summaries/' + filename, 'w')
summary_file.write(summary)
summary_file.close()
print("-")
def summarize_all():
# retrieve each of the articles
articles = os.listdir("articles")
for article in articles:
print('Reading articles/' + article)
article_file = io.open('articles/' + article, 'r')
text = article_file.read()
keyphrases = extract_key_phrases(text)
summary = extract_sentences(text)
write_files(summary, keyphrases, article)