Op-The-Urn-Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 0. Prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%pylab inline\n",
    "plt.style.use('ggplot')\n",
    "\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import multiprocessing\n",
    "from collections import Counter\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from scipy.stats import pearsonr\n",
    "from sklearn.metrics import classification_report, f1_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "\n",
    "text_path = '../CHICAGO_CORPUS/CHICAGO_NOVEL_CORPUS/'\n",
    "meta_df = pd.read_csv('../CHICAGO_CORPUS/CHICAGO_NOVEL_CORPUS_METADATA/CHICAGO_CORPUS_NOVELS.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Corpus Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# SAMPLE; RUN ONE TIME ONLY\n",
    "\n",
    "\n",
    "sent_dexs, parg_dexs, novl_dexs, ctrl_dexs = [], [], [], []\n",
    "for date in range(1901,2001):\n",
    "    temp_df = meta_df[meta_df['PUBL_DATE']==date]\n",
    "    dexs = temp_df.sample(16).index.tolist()\n",
    "    sent_dexs += dexs[:4]\n",
    "    parg_dexs += dexs[4:8]\n",
    "    novl_dexs += dexs[8:12]\n",
    "    ctrl_dexs += dexs[12:16]\n",
    "sample_dexs = sent_dexs + parg_dexs + novl_dexs + ctrl_dexs\n",
    "sample_dexs.sort()\n",
    "subset = ['SENT' if dex in sent_dexs else 'PARG' if dex in parg_dexs else 'NOVL'\\\n",
    "          if dex in novl_dexs else 'CTRL' for dex in sample_dexs]\n",
    "sample_df = meta_df.loc[sample_dexs]\n",
    "sample_df['SUBSET'] = subset\n",
    "sample_df.to_csv('sample_df.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pre-Process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_df = pd.read_csv('sample_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "texts = []\n",
    "for fname in sample_df['FILENAME']:\n",
    "    with open(text_path+fname,'r') as file_in:\n",
    "        texts.append(file_in.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "segmented_texts = []\n",
    "\n",
    "for i in range(len(texts)):\n",
    "    print(i)\n",
    "    this_text = texts[i]\n",
    "    paragraph_list = this_text.split('\\n\\n')\n",
    "    \n",
    "    tokenized_paragraphs = []\n",
    "    \n",
    "    for j in range(len(paragraph_list)):\n",
    "        this_paragraph = paragraph_list[j]\n",
    "        sentence_list = sent_tokenize(this_paragraph)\n",
    "        \n",
    "        tokenized_sentences = []\n",
    "        \n",
    "        for k in range(len(sentence_list)):\n",
    "            this_sentence = sentence_list[k]\n",
    "            word_list = word_tokenize(this_sentence)\n",
    "            \n",
    "            tokenized_sentences.append(word_list)\n",
    "        \n",
    "        tokenized_paragraphs.append(tokenized_sentences)\n",
    "    \n",
    "    segmented_texts.append(tokenized_paragraphs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# clear memory\n",
    "del texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "initial, final = [], []\n",
    "subset = list(sample_df['SUBSET'])\n",
    "\n",
    "for i in range(len(subset)):\n",
    "    if i%10 == 0:\n",
    "        print(i)\n",
    "    \n",
    "    text = segmented_texts[i]\n",
    "    \n",
    "    text_tokens = []\n",
    "    this_par_first_half, this_par_secnd_half = [], []\n",
    "    this_sent_first_half, this_sent_secnd_half = [], []\n",
    "    \n",
    "    for paragraph in text:\n",
    "        paragraph_tokens = []\n",
    "        \n",
    "        for sentence in paragraph:\n",
    "            sentence_tokens = []\n",
    "            \n",
    "            for word in sentence:\n",
    "                text_tokens.append(word)\n",
    "                paragraph_tokens.append(word)\n",
    "                sentence_tokens.append(word)\n",
    "                \n",
    "            this_sent_first_half += sentence_tokens[:len(sentence_tokens)//2]\n",
    "            this_sent_secnd_half += sentence_tokens[len(sentence_tokens)//2:]\n",
    "                \n",
    "        this_par_first_half += paragraph_tokens[:len(paragraph_tokens)//2]\n",
    "        this_par_secnd_half += paragraph_tokens[len(paragraph_tokens)//2:]\n",
    "    \n",
    "    if subset[i]=='NOVL':\n",
    "        initial.append(\" \".join(text_tokens[:len(text_tokens)//2]))\n",
    "        final.append(\" \".join(text_tokens[len(text_tokens)//2:]))\n",
    "    \n",
    "    elif subset[i]=='PARG':\n",
    "        initial.append(\" \".join(this_par_first_half))\n",
    "        final.append(\" \".join(this_par_secnd_half))\n",
    "        \n",
    "    elif subset[i]=='SENT':\n",
    "        initial.append(\" \".join(this_sent_first_half))\n",
    "        final.append(\" \".join(this_sent_secnd_half))\n",
    "        \n",
    "    elif subset[i]=='CTRL':\n",
    "        rand_tokens = np.random.permutation(text_tokens)\n",
    "        initial.append(\" \".join(rand_tokens[:len(rand_tokens)//2]))\n",
    "        final.append(\" \".join(rand_tokens[len(rand_tokens)//2:]))\n",
    "        \n",
    "    else:\n",
    "        print('broke',i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# clear memory\n",
    "del segmented_texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_df['INITIAL'] = initial\n",
    "sample_df['FINAL'] = final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# clear memory\n",
    "del initial\n",
    "del final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_df.to_pickle('sample_df_init_final.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Parameter Search: Ten-Fold Cross-Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_df = pd.read_pickle('sample_df_init_final.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SENT-PARG-NOVL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def master_function_cv(percentile):\n",
    "\n",
    "    test_auths = auth_list[k*len(auth_list)//10:(k+1)*len(auth_list)//10]\n",
    "    train_auths = [auth for auth in auth_list if auth not in test_auths]\n",
    "\n",
    "    train_df = spn_df[spn_df['AUTH_ID'].isin(train_auths)]\n",
    "    test_df = spn_df[spn_df['AUTH_ID'].isin(test_auths)]\n",
    "\n",
    "    train_labels = [0]*len(train_df)+[1]*len(train_df)\n",
    "    test_labels = [0]*len(test_df)+[1]*len(test_df)\n",
    "\n",
    "    tv = TfidfVectorizer(stop_words='english', max_features = num_feats, use_idf=False, norm='l1')\n",
    "    dtm_train = tv.fit_transform(list(train_df['INITIAL'])+list(train_df['FINAL'])).toarray()\n",
    "    dtm_test = tv.transform(list(test_df['INITIAL'])+list(test_df['FINAL'])).toarray()\n",
    "\n",
    "    sc = StandardScaler()\n",
    "    dtm_train_norm = sc.fit_transform(dtm_train)\n",
    "    dtm_test_norm = sc.transform(dtm_test)\n",
    "\n",
    "    lr = LogisticRegression(C=reg_coef)\n",
    "    lr.fit(dtm_train_norm, train_labels)\n",
    "    predictions = lr.predict(dtm_test_norm)\n",
    "    \n",
    "    return [predictions, test_labels]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pool = multiprocessing.Pool(10, maxtasksperchild=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "spn_df = sample_df[sample_df['SUBSET'].isin(['SENT', 'PARG', 'NOVL'])]\n",
    "auth_list = list(set(spn_df['AUTH_ID']))\n",
    "\n",
    "num_list = [2000,3000,4000,5000]\n",
    "coef_list = [1,0.1,0.01,0.001]\n",
    "\n",
    "f1_array = np.empty([len(num_list),len(coef_list)])\n",
    "\n",
    "for i in range(len(num_list)):\n",
    "    for j in range(len(coef_list)):\n",
    "        all_preds, all_labels = [], []\n",
    "        auth_list = np.random.permutation(auth_list)\n",
    "        num_feats = num_list[i]\n",
    "        reg_coef = coef_list[j]\n",
    "        percentiles = [x for x in range(10)]\n",
    "        \n",
    "        output = pool.map(master_function_cv, percentiles)\n",
    "        \n",
    "        for predictions, test_labels in output:\n",
    "\n",
    "            all_preds += list(predictions)\n",
    "            all_labels += test_labels\n",
    "\n",
    "        f1_array[i][j] = f1_score(all_preds, all_labels, average='weighted')\n",
    "        print(num_feats, reg_coef, f1_score(all_preds, all_labels, average='weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pool.close()\n",
    "pool.terminate()\n",
    "pool.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(f1_array)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Leave-One-Out Predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SENT-PARG-NOVL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def master_function_loocv(author):\n",
    "\n",
    "    train_df = spn_df[spn_df['AUTH_ID']!=author]\n",
    "    test_df = spn_df[spn_df['AUTH_ID']==author]\n",
    "\n",
    "    train_labels = [0]*len(train_df)+[1]*len(train_df)\n",
    "    test_labels = [0]*len(test_df)+[1]*len(test_df)\n",
    "\n",
    "    tv = TfidfVectorizer(stop_words='english', max_features = num_feats, use_idf=False, norm='l1')\n",
    "    dtm_train = tv.fit_transform(list(train_df['INITIAL'])+list(train_df['FINAL'])).toarray()\n",
    "    dtm_test = tv.transform(list(test_df['INITIAL'])+list(test_df['FINAL'])).toarray()\n",
    "\n",
    "    sc = StandardScaler()\n",
    "    dtm_train_norm = sc.fit_transform(dtm_train)\n",
    "    dtm_test_norm = sc.transform(dtm_test)\n",
    "\n",
    "    lr = LogisticRegression(C=reg_coef)\n",
    "    lr.fit(dtm_train_norm, train_labels)\n",
    "    predictions = lr.predict(dtm_test_norm)\n",
    "    \n",
    "    return [predictions, test_df.index.tolist()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "spn_df = sample_df[sample_df['SUBSET'].isin(['SENT', 'PARG', 'NOVL'])]\n",
    "auth_list = list(set(spn_df['AUTH_ID']))\n",
    "all_outputs = []\n",
    "\n",
    "num_feats = 3000\n",
    "reg_coef = 0.001\n",
    "\n",
    "for i in range(81):\n",
    "    print(i)\n",
    "    ten_auths = auth_list[i*10:(i+1)*10]\n",
    "    \n",
    "    pool = multiprocessing.Pool(10, maxtasksperchild=1)\n",
    "    output = pool.map(master_function_loocv, ten_auths)\n",
    "    \n",
    "    pool.close()\n",
    "    pool.terminate()\n",
    "    pool.join()\n",
    "    \n",
    "    all_outputs += output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "initial_preds, final_preds = [], []\n",
    "dex_list = spn_df.index.tolist()\n",
    "\n",
    "for predictions, dexs in all_outputs:\n",
    "    for i in range(len(dexs)):\n",
    "        this_dex = dexs[i]\n",
    "        this_initial_pred = predictions[i]\n",
    "        this_final_pred = predictions[i+len(dexs)]\n",
    "        \n",
    "        initial_preds.append([dex_list.index(this_dex),this_dex,this_initial_pred])\n",
    "        final_preds.append([dex_list.index(this_dex),this_dex,this_final_pred])\n",
    "\n",
    "initial_preds = sorted(initial_preds, key=lambda x: x[0], reverse=False)\n",
    "final_preds = sorted(final_preds, key=lambda x: x[0], reverse=False)\n",
    "\n",
    "initial_preds = [z for x,y,z in initial_preds]\n",
    "final_preds = [z for x,y,z in final_preds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "spn_df['INIT_PREDICT'] = initial_preds\n",
    "spn_df['FINAL_PREDICT'] = final_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "f1_score(list(spn_df['INIT_PREDICT'])+list(spn_df['FINAL_PREDICT']),\\\n",
    "         [0]*len(spn_df)+[1]*len(spn_df), average='weighted')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for subset in ['SENT', 'PARG', 'NOVL']:\n",
    "    sub_inits = list(spn_df[spn_df['SUBSET']==subset]['INIT_PREDICT'])\n",
    "    sub_finals = list(spn_df[spn_df['SUBSET']==subset]['FINAL_PREDICT'])\n",
    "    true_labels = [0]*len(sub_inits) + [1]*len(sub_finals)\n",
    "    this_f1 = f1_score(sub_inits+sub_finals, true_labels, average='weighted')\n",
    "    print(subset, this_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "these_columns = ['BOOK_ID', 'PUBL_DATE', 'SUBSET', 'INIT_PREDICT', 'FINAL_PREDICT']\n",
    "score_df = spn_df[these_columns]\n",
    "score_df.to_csv('spn_loocv_score.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Full Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "spn_df = sample_df[sample_df['SUBSET'].isin(['SENT', 'PARG', 'NOVL'])]\n",
    "labels = [0]*len(spn_df)+[1]*len(spn_df)\n",
    "\n",
    "num_feats = 3000\n",
    "reg_coef = 0.001\n",
    "\n",
    "tv = TfidfVectorizer(stop_words='english', max_features = num_feats, use_idf=False, norm='l1')\n",
    "dtm = tv.fit_transform(list(spn_df['INITIAL'])+list(spn_df['FINAL'])).toarray()\n",
    "dtm_ctrl = tv.transform(list(sample_df[sample_df['SUBSET']=='CTRL'][\"INITIAL\"])).toarray()\n",
    "\n",
    "sc = StandardScaler()\n",
    "dtm_norm = sc.fit_transform(dtm)\n",
    "dtm_ctrl_norm = sc.transform(dtm_ctrl)\n",
    "\n",
    "lr = LogisticRegression(C=reg_coef)\n",
    "lr.fit(dtm_norm, labels)\n",
    "predictions = lr.predict_proba(dtm_ctrl_norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "min_columns = ['BOOK_ID', 'PUBL_DATE', 'SUBSET']\n",
    "ctrl_score_df = sample_df[sample_df['SUBSET']=='CTRL'][min_columns]\n",
    "ctrl_score_df['P(INIT)_CTRL'] = predictions[:,0]\n",
    "ctrl_score_df.to_csv('ctrl_score.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_weight_df = pd.DataFrame()\n",
    "feature_weight_df['FEAT_NAME'] = tv.get_feature_names()\n",
    "feature_weight_df['WEIGHT'] = lr.coef_[0]\n",
    "feature_weight_df.to_csv('feature_weights.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pickle.dump( [tv,sc,lr], open( 'tv_sc_lr.p', 'wb' ) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Hypothesis Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xs = ctrl_score_df['PUBL_DATE']\n",
    "ys = ctrl_score_df['P(INIT)_CTRL']\n",
    "fit = np.polyfit(xs, ys, deg=1)\n",
    "\n",
    "figure(figsize=(10,10))\n",
    "scatter(xs, ys, color='c')\n",
    "plot(xs, fit[0] * xs + fit[1], color='m')\n",
    "xlabel('Publication Date')\n",
    "ylabel('Probability of \"Initial-ness\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pearsonr(ctrl_score_df['PUBL_DATE'], ctrl_score_df['P(INIT)_CTRL'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}