-
Notifications
You must be signed in to change notification settings - Fork 0
/
potus_pipe_1_gensimlda_01_create_lda.py
174 lines (133 loc) · 4.93 KB
/
potus_pipe_1_gensimlda_01_create_lda.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
# %%
import helpers.process as process
from helpers import io as pickle_io
from gensim.models import ldamodel
import tqdm
from helpers import potus
# from gensim.models import CoherenceModel
# %%
# Set up log to external log file
# import logging
# logging.basicConfig(filename='lda_model.log', format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
# Set up log to terminal
import logging
logging.basicConfig(format=logging.BASIC_FORMAT, level=logging.INFO)
# %%
import argparse
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('--experiment', help='experiment folder name', default='tempexp1', required=False)
arg_parser.add_argument('--num_topics', help='number of topics for modelling', default=100, type=int, required=False)
arg_parser.add_argument('--iterations', help='number of modelling iterations', default=50, type=int, required=False)
arg_parser.add_argument('--passes', help='number of passes for modelling', default=4, type=int, required=False)
arg_parser.add_argument("-f", "--fff", help="a dummy argument to fool ipython", default="1")
args = arg_parser.parse_args()
expfolder = args.experiment
num_topics = args.num_topics
iterations = args.iterations
passes = args.passes
print('experiment folder name: ', expfolder)
print('number of topics: ', num_topics)
print('iterations: ', iterations)
print('passes: ', passes)
# %%
root_folder = potus.rootFolder(expfolder)
print('root_folder: ', root_folder)
bigrams = pickle_io.from_pickle(root_folder + '/bigrams.pkl')
# # %%
# this logic is move to the begining of the pipeline before the speeches df is saved to file
# min_speech_len = 10
# bigrams = [b for b in bigrams if len(b) >= min_speech_len]
# %%
speeches_dictionary = process.make_dictionary(bigrams, large_list=True)
# %%
# AJ
# Filter out words that occur less than <no_below> documents, or more than <no_above> x 100% of the documents.
speeches_dictionary.filter_extremes(no_below=10, no_above=0.5)
# %%
bow = [
speeches_dictionary.doc2bow(t.split(','))
for t in tqdm.tqdm(bigrams, desc='Creating BOW')]
bow_fn = root_folder + '/bow.pkl'
print(f'save bow to {bow_fn}')
pickle_io.to_pickle(bow, bow_fn)
# %% [markdown]
# # Build the topic model using gensim LDA model
print('fit ldamodel')
speeches_lda = ldamodel.LdaModel(
corpus=bow, # my list of BOW
id2word=speeches_dictionary,
# dictionary for resolving ids from corpus to term in vocablulary
num_topics=num_topics,
iterations=iterations,
passes=passes,
chunksize=100,
update_every=2,
alpha='auto',
eta='auto',
per_word_topics=True,
random_state=100
)
# %%
lda_model_fn = root_folder + '/lda.model'
print(f'save lda model to file {lda_model_fn}')
speeches_lda.save(lda_model_fn)
# %%
# AJ
import numpy as np
top_topics = speeches_lda.top_topics(bow)
# Average topic coherence is the sum of topic coherences of all topics, divided by the number of topics.
coherences = [t[1] for t in top_topics]
topic_coherence_mean = sum(coherences) / num_topics
topic_coherence_25th = np.quantile(coherences, 0.25)
topic_coherence_50th = np.quantile(coherences, 0.50)
topic_coherence_75th = np.quantile(coherences, 0.75)
print('Average topic coherence: %.4f.' % topic_coherence_mean)
# Topic diversity: what proportion of the top words across each topic are unique to that topic?
top_terms = []
for x in top_topics:
for y in x[0]:
top_terms.append(y[1])
num_terms = len(top_terms)
num_unique = len(set(top_terms))
print('Top topic terms: ' + str(num_terms) + ', of which ' + str(num_unique) +' are unique.')
topic_diversity = num_unique / num_terms
print('Topic diversity score: ' + str(topic_diversity))
from pprint import pprint
pprint(top_topics)
# now save the results to a log pkl
import pickle
log_filename = '/home/azureuser/cloudfiles/code/data/processing/potus/experiment/lda_eval_log.pkl'
try:
eval_log = pickle_io.from_pickle(log_filename)
except (OSError, IOError) as e:
eval_log = []
new_eval_log_entry = {'experiment': expfolder,
'num_topics': num_topics, 'iterations': iterations, 'passes': passes,
'topic_coherence_mean': topic_coherence_mean,
'topic_coherence_25th': topic_coherence_25th,
'topic_coherence_50th': topic_coherence_50th,
'topic_coherence_75th': topic_coherence_75th,
'topic_diversity': topic_diversity}
eval_log.append(new_eval_log_entry)
pickle_io.to_pickle(eval_log, log_filename)
# %%
# # Compute Perplexity
# print(
# '\nPerplexity: ',
# speeches_lda.log_perplexity(bow))
# # a measure of how good the model is. lower the better.
# %%
# Compute Coherence Score
# coherence_model_lda = CoherenceModel(
# model=speeches_lda,
# texts=list(speeches_df.words_and_bigrams),
# dictionary=speeches_dictionary,
# coherence='c_v')
# coherence_lda = coherence_model_lda.get_coherence()
# print('\nCoherence Score: ', coherence_lda)
# # %%
# coherence_model_lda.top_topics_as_word_lists(
# speeches_lda,
# dictionary=speeches_dictionary)[4]
# %%
speeches_lda.print_topics()