In [1]:
%matplotlib inline
import matplotlib.pyplot as plt
In [2]:
import pandas as pd
In [5]:
df = pd.read_csv('/Users/sara/github/data/tianya_bbs_threads_list.txt',\
sep = "\t", names = ['title','link', \
'author','author_page',\
'click','reply','time'])
df[:2]
Out[5]:
In [6]:
# df=df.rename(columns = {0:'title', 1:'link', \
# 2:'author',3:'author_page',\
# 4:'click', 5:'reply', 6:'time'})
# df[:5]
In [8]:
da = pd.read_csv('/Users/sara/github/data/tianya_bbs_threads_author_info.txt',
sep = "\t", names = ['author_page','followed_num',\
'fans_num','post_num', \
'comment_num'])
da[:2]
Out[8]:
In [9]:
# da=da.rename(columns = {0:'author_page', 1:'followed_num',\
# 2:'fans_num',3:'post_num', \
# 4:'comment_num'})
# # da[:5]
In [11]:
data = pd.concat([df,da], axis=1)
len(data)
Out[11]:
In [12]:
data[:3]
Out[12]:
In [13]:
type(data.time[0])
Out[13]:
In [14]:
# extract date from datetime
date = map(lambda x: x[:10], data.time)
#date = [i[:10] for i in data.time]
data['date'] = pd.to_datetime(date)
In [15]:
# convert str to datetime format
data.time = pd.to_datetime(data.time)
data['month'] = data.time.dt.month
data['year'] = data.time.dt.year
data['day'] = data.time.dt.day
type(data.time[0])
Out[15]:
In [16]:
data.describe()
Out[16]:
In [17]:
import statsmodels.api as sm
In [18]:
' '.join(dir(sm))
Out[18]:
In [19]:
' '.join(dir(sm.stats))
Out[19]:
In [20]:
data.describe()
Out[20]:
In [21]:
import numpy as np
np.mean(data.click), np.std(data.click), np.sum(data.click)
Out[21]:
In [22]:
# 不加权的变量描述
d1 = sm.stats.DescrStatsW(data.click, \
weights=[1 for i in data.click])
d1.mean, d1.var, d1.std, d1.sum
Out[22]:
In [23]:
# 加权的变量描述
d1 = sm.stats.DescrStatsW(data.click, weights=data.reply)
d1.mean, d1.var, d1.std, d1.sum
Out[23]:
In [24]:
np.median(data.click) # np.percentile
Out[24]:
In [25]:
plt.hist(data.click)
plt.show()
In [26]:
plt.hist(data.reply, color = 'green')
plt.show()
In [27]:
plt.hist(np.log(data.click+1), color='green')
plt.hist(np.log(data.reply+1), color='red')
plt.show()
In [28]:
# Plot the height and weight to see
plt.boxplot([np.log(data.click+1)])
plt.show()
In [29]:
# Plot the height and weight to see
plt.boxplot([data.click, data.reply])
plt.show()
In [30]:
def transformData(dat):
results = []
for i in dat:
if i != 'na':
results.append( int(i))
else:
results.append(0)
return results
In [32]:
data.fans_num = transformData(data.fans_num)
data.followed_num = transformData(data.followed_num )
data.post_num = transformData(data.post_num )
data.comment_num = transformData(data.comment_num )
data.describe()
Out[32]:
In [33]:
import numpy as np
# Plot the height and weight to see
plt.boxplot([np.log(data.click+1), np.log(data.reply+1),
np.log(data.fans_num+1),\
np.log(data.followed_num + 1)],
labels = ['$Click$', '$Reply$', '$Fans$',\
'$Followed$'])
plt.show()
In [34]:
fig = plt.figure(figsize=(12,4))
data.boxplot(return_type='dict')
plt.yscale('log')
plt.show()
In [35]:
' '.join(dir(data)[200:])
Out[35]:
In [36]:
from pandas.tools import plotting
# fig = plt.figure(figsize=(10, 10))
plotting.scatter_matrix(data[['click', 'reply',\
'post_num','comment_num']])
plt.show()
In [37]:
' '.join(dir(plotting))
Out[37]:
In [38]:
import seaborn # conda install seaborn
seaborn.pairplot(data, vars=['click', 'reply', \
'post_num', 'comment_num'],
kind='reg')
Out[38]:
In [42]:
seaborn.lmplot(y='reply', x='click', data=data,
size = 5)
plt.show()
In [43]:
data.year.value_counts()
Out[43]:
In [44]:
d = data.year.value_counts()
dd = pd.DataFrame(d)
dd = dd.sort_index(axis=0, ascending=True)
dd
Out[44]:
In [45]:
dd.index
Out[45]:
In [46]:
dd_date_str = map(lambda x: str(x) +'-01-01', dd.index)
dd_date_str
Out[46]:
In [47]:
dd_date = pd.to_datetime(dd_date_str)
dd_date
Out[47]:
In [48]:
plt.plot(dd_date, dd.year, 'r-o')
plt.show()
In [49]:
ds = dd.cumsum()
ds
Out[49]:
In [84]:
d = data.year.value_counts()
dd = pd.DataFrame(d)
dd = dd.sort_index(axis=0, ascending=True)
ds = dd.cumsum()
def getDate(dat):
dat_date_str = map(lambda x: str(x) +'-01-01', dat.index)
dat_date = pd.to_datetime(dat_date_str)
return dat_date
ds.date = getDate(ds)
dd.date = getDate(dd)
plt.plot(ds_date, ds.year, 'g-s', label = '$Cumulative\: Number\:of\: Threads$')
plt.plot(dd_date, dd.year, 'r-o', label = '$Yearly\:Number\:of\:Threads$')
plt.legend(loc=2,numpoints=1,fontsize=13)
plt.show()
In [51]:
dg = data.groupby('year').sum()
dg
Out[51]:
In [54]:
dgs = dg.cumsum()
dgs
Out[54]:
In [55]:
def getDate(dat):
dat_date_str = map(lambda x: str(x) +'-01-01', dat.index)
dat_date = pd.to_datetime(dat_date_str)
return dat_date
dg.date = getDate(dg)
In [56]:
fig = plt.figure(figsize=(12,5))
plt.plot(dg.date, dg.click, 'r-o', label = '$Yearly\:Number\:of\:Clicks$')
plt.plot(dg.date, dg.reply, 'g-s', label = '$Yearly\:Number\:of\:Replies$')
plt.plot(dg.date, dg.fans_num, 'b->', label = '$Yearly\:Number\:of\:Fans$')
plt.yscale('log')
plt.legend(loc=4,numpoints=1,fontsize=13)
plt.show()
In [57]:
data.groupby('year')['click'].sum()
Out[57]:
In [58]:
data.groupby('year')['click'].mean()
Out[58]:
In [59]:
repost = []
for i in df.title:
if u'转载' in i.decode('utf8'):
repost.append(1)
else:
repost.append(0)
In [60]:
df['repost'] = repost
In [61]:
df.groupby('repost').mean()
Out[61]:
In [62]:
df['click'][df['repost']==0][:5]
Out[62]:
In [63]:
df['click'][df['repost']==1][:5]
Out[63]:
In [64]:
from scipy import stats
stats.ttest_ind(data.click, data.repost)
In [65]:
sm.stats.ttest_ind(data.click, data.reply)
# test statistic, pvalue and degrees of freedom
Out[65]:
In [66]:
from scipy.stats import chisquare
chisquare([16, 18, 16, 14, 12, 12], \
f_exp=[16, 16, 16, 16, 16, 8])
Out[66]:
In [67]:
from scipy.stats import chisqprob, chi2
# p_value = chi2.sf(chi_statistic, df)
print chisqprob(3.94,1), 1 - chi2.cdf(3.94,1)
In [68]:
print np.corrcoef(data.click, data.reply)
print np.corrcoef(np.log(data.click+1), \
np.log(data.reply+1))
In [69]:
data.corr()
Out[69]:
In [70]:
plt.plot(df.click, df.reply, 'r-o')
plt.show()
In [71]:
plt.plot(df.click, df.reply, 'gs')
plt.xlabel('$Clicks$', fontsize = 20)
plt.ylabel('$Replies$', fontsize = 20)
plt.xscale('log')
plt.yscale('log')
plt.title('$Allowmetric\,Law$', fontsize = 20)
plt.show()
In [72]:
import numpy as np
import statsmodels.api as sm
import statsmodels.formula.api as smf
In [73]:
# Load data
dat = sm.datasets.get_rdataset("Guerry", "HistData").data
# Fit regression model (using the natural log of one of the regressors)
results = smf.ols('Lottery ~ Literacy + np.log(Pop1831)', \
data=dat).fit()
In [74]:
# Inspect the results
print results.summary()
In [75]:
reg = smf.ols('reply ~ click + followed_num', \
data=data).fit()
In [76]:
print reg.summary()
In [77]:
reg1 = smf.ols('np.log(reply+1) ~ np.log(click+1) \
+np.log(followed_num+1)+month', data=data).fit()
print reg1.summary()
In [78]:
fig = plt.figure(figsize=(12,8))
fig = sm.graphics.plot_partregress_grid(reg1, fig = fig)
plt.show()
In [79]:
import statsmodels.api as sm
from statsmodels.formula.api import ols
moore = sm.datasets.get_rdataset("Moore", "car",
cache=True) # load data
data = moore.data
data = data.rename(columns={"partner.status" :
"partner_status"}) # make name pythonic
In [80]:
data[:5]
Out[80]:
In [81]:
moore_lm = ols('conformity ~ C(fcategory, Sum)*C(partner_status, Sum)',
data=data).fit()
In [82]:
fig = plt.figure(figsize=(12,8))
fig = sm.graphics.plot_partregress_grid(moore_lm, fig = fig)
plt.show()
In [83]:
table = sm.stats.anova_lm(moore_lm, typ=2) # Type 2 ANOVA DataFrame
print table