In [2]:
%matplotlib inline
import matplotlib.pyplot as plt
import pandas as pd
df = pd.read_csv('/Users/xiha/github/cjc-gh-pages/data/tianya_bbs_threads_list.txt', sep = "\t", header=None)
df[:2]
Out[2]:
In [3]:
df=df.rename(columns = {0:'title', 1:'link', 2:'author',3:'author_page', 4:'click', 5:'reply', 6:'time'})
df[:5]#重新进行header的命名
Out[3]:
In [4]:
da = pd.read_csv('/Users/xiha/github/cjc-gh-pages/data/tianya_bbs_threads_author_info.txt', sep = "\t", header=None)
da[:2]
Out[4]:
In [5]:
da=da.rename(columns = {0:'author_page', 1:'followed_num', 2:'fans_num',3:'post_num', 4:'comment_num'})
da[:5]
Out[5]:
In [6]:
data = pd.concat([df,da], axis=1)#合并数据集
len(data)
Out[6]:
Time¶
In [8]:
type(data.time[0])#字符型的数据
Out[8]:
In [9]:
# extract date from datetime
date = map(lambda x: x[:10], data.time)#map(函数:参数情况)这个lambda函数运行效率高，处理大数据的时候用！
data['date'] = pd.to_datetime(date)#把日期格式的字符串转换成日期类型的数据
In [10]:
# convert str to datetime format
data.time = pd.to_datetime(data.time)
data['month'] = data.time.dt.month
data['year'] = data.time.dt.year
data['day'] = data.time.dt.day
type(data.time[0])
Out[10]:
In [11]:
data[:3]#显示前三个记录
data.describe()
Out[11]:
Statsmodels¶
In [12]:
import statsmodels.api as sm
In [13]:
' '.join(dir(sm.stats))
Out[13]:
Describe¶
In [14]:
data.describe()
Out[14]:
In [15]:
import numpy as np
np.mean(data.click), np.std(data.click), np.sum(data.click)
Out[15]:
In [16]:
# 不加权的变量描述
d1 = sm.stats.DescrStatsW(data.click, weights=[1 for i in data.click])
d1.mean, d1.var, d1.std, d1.sum
Out[16]:
In [17]:
# 加权的变量描述
d1 = sm.stats.DescrStatsW(data.click, weights=data.reply)
d1.mean, d1.var, d1.std, d1.sum
Out[17]:
In [18]:
np.median(data.click)
Out[18]:
In [19]:
plt.hist(data.click)
plt.show()
In [23]:
plt.hist(data.reply, color = 'purple')
plt.show()
In [26]:
plt.hist(np.log(data.click+1), color='green')
plt.hist(np.log(data.reply+1), color='purple')
plt.show()
In [27]:
# Plot the height and weight to see
plt.boxplot([np.log(data.click+1)])
plt.show()
In [28]:
# Plot the height and weight to see
plt.boxplot([data.click, data.reply])
plt.show()
In [29]:
def transformData(dat):#对一个变量进行处理
results = []
for i in dat:
if i != 'na':
results.append( int(i))
else:
results.append(0)
return results
In [30]:
data.fans_num = transformData(data.fans_num)
data.followed_num = transformData(data.followed_num )
data.post_num = transformData(data.post_num )
data.comment_num = transformData(data.comment_num )
data.describe()
Out[30]:
In [31]:
# Plot the height and weight to see
plt.boxplot([np.log(data.click+1), np.log(data.reply+1),
np.log(data.fans_num+1), np.log(data.followed_num + 1)],
labels = ['$Click$', '$Reply$', '$Fans$', '$Followed$'])
plt.show()
Pandas自身已经包含了boxplot的功能¶
In [32]:
fig = plt.figure(figsize=(12,4))
data.boxplot(return_type='dict')
plt.yscale('log')
plt.show()
In [33]:
from pandas.tools import plotting
#fig = plt.figure(figsize=(10, 10))
plotting.scatter_matrix(data[['click', 'reply', 'post_num','comment_num']])
plt.show()
In [34]:
import seaborn # conda install seaborn
seaborn.pairplot(data, vars=['click', 'reply', 'post_num', 'comment_num'],
kind='reg')
Out[34]:
In [35]:
seaborn.pairplot(data, vars=['click', 'reply', 'post_num'],
kind='reg', hue='year')
Out[35]:
In [36]:
seaborn.lmplot(y='reply', x='click', data=data)
Out[36]:
values_counts¶
In [37]:
data.year.value_counts()#每一年的贴子有
Out[37]:
In [38]:
d = data.year.value_counts()
dd = pd.DataFrame(d)
dd = dd.sort_index(axis=0, ascending=True)
dd
Out[38]:
In [39]:
dd.index
Out[39]:
In [40]:
dd_date_str = map(lambda x: str(x) +'-01-01', dd.index)
dd_date_str
Out[40]:
In [41]:
dd_date = pd.to_datetime(dd_date_str)#pd里有处理日期的函数
dd_date
Out[41]:
In [42]:
plt.plot(dd_date, dd.year, 'r-o')
plt.show()
In [43]:
ds = dd.cumsum()
ds
Out[43]:
In [44]:
d = data.year.value_counts()
dd = pd.DataFrame(d)
dd = dd.sort_index(axis=0, ascending=True)
ds = dd.cumsum()
def getDate(dat):
dat_date_str = map(lambda x: str(x) +'-01-01', dat.index)
dat_date = pd.to_datetime(dat_date_str)
return dat_date
ds_date = getDate(ds)
dd_date = getDate(dd)
plt.plot(ds_date, ds.year, 'g-s', label = '$Cumulative\: Number\:of\: Threads$')
plt.plot(dd_date, dd.year, 'r-o', label = '$Yearly\:Number\:of\:Threads$')
plt.legend(loc=2,numpoints=1,fontsize=13)
plt.show()
groupby¶
In [45]:
dg = data.groupby('year').sum()
dg
Out[45]:
In [46]:
dgs = dg.cumsum()
dgs
Out[46]:
In [47]:
def getDate(dat):
dat_date_str = map(lambda x: str(x) +'-01-01', dat.index)
dat_date = pd.to_datetime(dat_date_str)
return dat_date
dg.date = getDate(dg)
In [48]:
fig = plt.figure(figsize=(12,5))
plt.plot(dg.date, dg.click, 'r-o', label = '$Yearly\:Number\:of\:Clicks$')
plt.plot(dg.date, dg.reply, 'g-s', label = '$Yearly\:Number\:of\:Replies$')
plt.plot(dg.date, dg.fans_num, 'b->', label = '$Yearly\:Number\:of\:Fans$')
plt.yscale('log')
plt.legend(loc=4,numpoints=1,fontsize=13)
plt.show()
In [49]:
data.groupby('year')['click'].sum()
Out[49]:
In [50]:
data.groupby('year')['click'].mean()
Out[50]:
常用的统计分析方法¶
- 检验
- 卡方检验
- 相关
- 回归
RQ: 转载的文章的点击量是否显著地小于原创的文章？¶
In [51]:
repost = []
for i in df.title:
if u'转载' in i.decode('utf8'):
repost.append(1)
else:
repost.append(0)
In [52]:
df['repost'] = repost
In [53]:
df.groupby('repost').sum()
Out[53]:
In [54]:
df['click'][df['repost']==0][:5]
Out[54]:
In [55]:
df['click'][df['repost']==1][:5]
Out[55]:
In [56]:
from scipy import stats
stats.ttest_ind(df.click, df.repost)
Out[56]:
In [57]:
sm.stats.ttest_ind(data.click, data.reply)
Out[57]:
A chi-squared test 卡方检验 看数据是不是相关联的？¶
In [58]:
from scipy.stats import chisquare
chisquare([16, 18, 16, 14, 12, 12], f_exp=[16, 16, 16, 16, 16, 8])#远高于，所以不能拒绝原假设，原假设成立
Out[58]:
In [59]:
from scipy.stats import chisqprob, chi2
# p_value = chi2.sf(chi_statistic, df)
print chisqprob(3.94,1), 1 - chi2.cdf(3.94,1)
Correlation 相关性分析¶
In [60]:
import numpy as np
print np.corrcoef(data.click, data.reply) #计算任意两个变量的相关性！！！
print np.corrcoef(np.log(data.click+1), np.log(data.reply+1))
In [61]:
data.corr()
Out[61]:
In [62]:
plt.plot(df.click, df.reply, 'r-o')#不取对数的情况下
plt.show()
In [63]:
plt.plot(df.click, df.reply, 'gs')#取对数之后
plt.xlabel('$Clicks$', fontsize = 20)
plt.ylabel('$Replies$', fontsize = 20)
plt.xscale('log')
plt.yscale('log')
plt.title('$Allowmetric\,Law$', fontsize = 20)
plt.show()
Regression 回归分析¶
- 线性回归，要求变量是连续性的，
- 多分类的，用groupby
In [64]:
import numpy as np
import statsmodels.api as sm
import statsmodels.formula.api as smf
In [65]:
# Load data
dat = sm.datasets.get_rdataset("Guerry", "HistData").data
# Fit regression model (using the natural log of one of the regressors)
results = smf.ols('Lottery ~ Literacy + np.log(Pop1831)', data=dat).fit()
#研究Lottery和Literacy + np.log(Pop1831的相关性
In [66]:
# Inspect the results
print results.summary()
In [67]:
reg = smf.ols('reply ~ click + followed_num', data=data).fit()#回复和点击和跟贴数量的关系
In [68]:
print reg.summary()
In [70]:
import statsmodels.api as sm
from statsmodels.formula.api import ols
moore = sm.datasets.get_rdataset("Moore", "car",
cache=True) # load data
data = moore.data
data = data.rename(columns={"partner.status" :
"partner_status"}) # make name pythonic
In [71]:
data[:5]
Out[71]:
In [72]:
moore_lm = ols('conformity ~ C(fcategory, Sum)*C(partner_status, Sum)',
data=data).fit()
In [73]:
fig = plt.figure(figsize=(12,8))
fig = sm.graphics.plot_partregress_grid(moore_lm, fig = fig)
plt.show()
In [74]:
table = sm.stats.anova_lm(moore_lm, typ=2) # Type 2 ANOVA DataFrame
print table
In [ ]: