In [2]:
with open('/Users/sara/github/data/gov_reports1954-2016.txt', 'r') as f:
reports = f.readlines()
In [3]:
len(reports)
Out[3]:
In [4]:
print reports[32][:1000]
In [5]:
%matplotlib inline
import matplotlib.cm as cm
import matplotlib.pyplot as plt
import sys
import numpy as np
from collections import defaultdict
import statsmodels.api as sm
from wordcloud import WordCloud
import jieba
import matplotlib
import gensim
from gensim import corpora, models, similarities
from gensim.utils import simple_preprocess
from gensim.parsing.preprocessing import STOPWORDS
matplotlib.rcParams['font.sans-serif'] = ['Microsoft YaHei'] #指定默认字体
matplotlib.rc("savefig", dpi=400)
In [6]:
# 为了确保中文可以在matplotlib里正确显示
matplotlib.rcParams['font.sans-serif'] = ['Microsoft YaHei'] #指定默认字体
# 需要确定系统安装了Microsoft YaHei
In [7]:
matplotlib.rcParams
Out[7]:
In [8]:
import jieba
seg_list = jieba.cut("我来到北京清华大学", cut_all=True)
print("Full Mode: " + "/ ".join(seg_list)) # 全模式
seg_list = jieba.cut("我来到北京清华大学", cut_all=False)
print("Default Mode: " + "/ ".join(seg_list)) # 精确模式
seg_list = jieba.cut("他来到了网易杭研大厦") # 默认是精确模式
print(", ".join(seg_list))
seg_list = jieba.cut_for_search("小明硕士毕业于中国科学院计算所，后在日本京都大学深造") # 搜索引擎模式
print(", ".join(seg_list))
In [10]:
filename = '/Users/sara/github/data/stopwords.txt'
stopwords = {}
f = open(filename, 'r')
line = f.readline().rstrip()
while line:
stopwords.setdefault(line, 0)
stopwords[line.decode('utf-8')] = 1
line = f.readline().rstrip()
f.close()
In [11]:
adding_stopwords = [u'我们', u'要', u'地', u'有', u'这', u'人',
u'发展',u'建设',u'加强',u'继续',u'对',u'等',u'推进',u'工作',u'增加']
for s in adding_stopwords: stopwords[s]=10
In [12]:
import jieba.analyse
txt = reports[-1]
tf = jieba.analyse.extract_tags(txt, topK=200, withWeight=True)
In [13]:
print u"、".join([i[0] for i in tf[:50]])
In [14]:
plt.hist([i[1] for i in tf])
plt.show()
In [15]:
tr = jieba.analyse.textrank(txt,topK=200, withWeight=True)
print u"、".join([i[0] for i in tr[:50]])
In [16]:
plt.hist([i[1] for i in tr])
plt.show()
In [17]:
import pandas as pd
def keywords(index):
txt = reports[-index]
tf = jieba.analyse.extract_tags(txt, topK=200, withWeight=True)
tr = jieba.analyse.textrank(txt,topK=200, withWeight=True)
tfdata = pd.DataFrame(tf, columns=['word', 'tfidf'])
trdata = pd.DataFrame(tr, columns=['word', 'textrank'])
worddata = pd.merge(tfdata, trdata, on='word')
plt.plot(worddata.tfidf, worddata.textrank, linestyle='',marker='.')
for i in range(len(worddata.word)):
plt.text(worddata.tfidf[i], worddata.textrank[i], worddata.word[i],
fontsize = worddata.textrank[i]*15, color = 'red', rotation = 0)
plt.title(txt[:4])
plt.xlabel('Tf-Idf')
plt.ylabel('TextRank')
plt.show()
In [18]:
keywords(1)
In [19]:
keywords(2)
In [20]:
keywords(3)
In [24]:
def wordcloudplot(txt, year):
wordcloud = WordCloud(font_path='/Users/sara/github/data/msyh.ttf').generate(txt)
# Open a plot of the generated image.
plt.imshow(wordcloud)
plt.title(year)
plt.axis("off")
#plt.show()
In [25]:
txt = reports[-1]
tfidf200= jieba.analyse.extract_tags(txt, topK=200, withWeight=False)
seg_list = jieba.cut(txt, cut_all=False)
seg_list = [i for i in seg_list if i in tfidf200]
txt200 = r' '.join(seg_list)
wordcloudplot(txt200, txt[:4])
In [28]:
wordfreq = defaultdict(int)
for i in seg_list:
wordfreq[i] +=1
wordfreq = [[i, wordfreq[i]] for i in wordfreq]
wordfreq.sort(key= lambda x:x[1], reverse = True )
print u"、 ".join([ i[0] + u'（' + str(i[1]) +u'）' for i in wordfreq ])
In [27]:
#jieba.add_word('股灾', freq=100, tag=None)
txt = reports[-1]
seg_list = jieba.cut(txt, cut_all=False)
seg_list = [i for i in seg_list if i not in stopwords]
txt = r' '.join(seg_list)
wordcloudplot(txt, txt[:4])
#file_path = '/Users/chengjun/GitHub/cjc2016/figures/wordcloud-' + txt[:4] + '.png'
#plt.savefig(file_path,dpi = 300, bbox_inches="tight",transparent = True)
In [29]:
#jieba.add_word('股灾', freq=100, tag=None)
for txt in reports:
seg_list = jieba.cut(txt, cut_all=False)
seg_list = [i for i in seg_list if i not in stopwords]
txt = r' '.join(seg_list)
wordcloudplot(txt, txt[:4])
file_path = '/Users/chengjun/GitHub/cjc2016/figure/wordcloud-' + txt[:4] + '.png'
plt.savefig(file_path,dpi = 400, bbox_inches="tight",transparent = True)
In [30]:
import jieba.analyse
wordset = []
for txt in reports:
top20= jieba.analyse.textrank(txt, topK=200, withWeight=False)
for w in top20:
if w not in wordset:
wordset.append(w)
In [31]:
len(wordset)
Out[31]:
In [32]:
print ' '.join(wordset)
In [33]:
from collections import defaultdict
data = defaultdict(dict)
years = [int(i[:4]) for i in reports]
for i in wordset:
for year in years:
data[i][year] = 0
In [34]:
for txt in reports:
year = int(txt[:4])
top1000= jieba.analyse.textrank(txt, topK=1000, withWeight=True)
for ww in top1000:
word, weight = ww
if word in wordset:
data[word][year]+= weight
In [35]:
word_weight = []
for i in data:
word_weight.append([i, np.sum(data[i].values())])
In [36]:
word_weight.sort(key= lambda x:x[1], reverse = True )
top50 = [i[0] for i in word_weight[:50]]
In [37]:
print ' '.join(top50)
In [38]:
def plotEvolution(word, color, linestyle, marker):
cx = data[word]
plt.plot(cx.keys(), cx.values(), color = color,
linestyle=linestyle, marker=marker, label= word)
plt.legend(loc=2,fontsize=8)
plt.ylabel(u'词语重要性')
In [39]:
plotEvolution(u'民主', 'g', '-', '>')
plotEvolution(u'法制', 'b', '-', 's')
In [40]:
plotEvolution(u'动能', 'b', '-', 's')
plotEvolution(u'互联网', 'g', '-', '>')
In [41]:
plotEvolution(u'工业', 'y', '-', '<')
plotEvolution(u'农业', 'r', '-', 'o')
plotEvolution(u'制造业', 'b', '-', 's')
plotEvolution(u'服务业', 'g', '-', '>')
In [42]:
plotEvolution(u'教育', 'r', '-', 'o')
plotEvolution(u'社会保障', 'b', '-', 's')
plotEvolution(u'医疗', 'g', '-', '>')
In [43]:
plotEvolution(u'环境', 'b', '-', 's')
plotEvolution(u'住房', 'purple', '-', 'o')
In [44]:
plotEvolution(u'发展', 'y', '-', '<')
plotEvolution(u'经济', 'r', '-', 'o')
plotEvolution(u'改革', 'b', '-', 's')
plotEvolution(u'创新', 'g', '-', '>')
In [45]:
plotEvolution(u'社会主义', 'r', '-', 'o')
plotEvolution(u'马克思主义', 'b', '-', 's')
In [ ]: