Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading dataset\n",
      "Total stances: 49972\n",
      "Total bodies: 1683\n",
      "Loading features for slave <class 'ensemble.FNCBaseLine.FNCBaseLine'>\n",
      "Loading features for slave <class 'ensemble.XiaoxuanWang.XiaoxuanWang'>\n",
      "Loading features for slave <class 'ensemble.JiashuPu.JiashuPu'>\n",
      "Loading features for slave <class 'ensemble.GiorgosMyrianthous.GiorgosMyrianthous'>\n",
      "Loading features for slave <class 'ensemble.MingjieChen.MingjieChen'>\n",
      "Embeddings: 208 x 300\n",
      "UPPER BOUND:::\n",
      "9437\n",
      "9622\n",
      "0.9807732280191228\n",
      "49\n",
      "49\n",
      "-------------------------------------------------------------\n",
      "|           |   agree   | disagree  |  discuss  | unrelated |\n",
      "-------------------------------------------------------------\n",
      "|   agree   |    507    |    10     |    219    |    26     |\n",
      "-------------------------------------------------------------\n",
      "| disagree  |    33     |    58     |    64     |     7     |\n",
      "-------------------------------------------------------------\n",
      "|  discuss  |    135    |    14     |   1586    |    65     |\n",
      "-------------------------------------------------------------\n",
      "| unrelated |     6     |     0     |    26     |   6866    |\n",
      "-------------------------------------------------------------\n",
      "Score: 3986.25 out of 4448.5\t(89.60885691806227%)\n",
      "Reading dataset\n",
      "Total stances: 25413\n",
      "Total bodies: 904\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "25413it [00:29, 863.22it/s]\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Input contains NaN, infinity or a value too large for dtype('float64').",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-365be87aeb2e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    125\u001b[0m         \u001b[0mslave\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprepare_final\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtest_dataset\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mall_folds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    126\u001b[0m         \u001b[0mslave\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpreload_features\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_dataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstances\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"test.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 127\u001b[0;31m         \u001b[0mslv_predicted_test\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mLABELS\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mslave\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_dataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstances\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    128\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m     \u001b[0mfinal_predictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmaster\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_dataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstances\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mslv_predicted_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/james/Dropbox/phd/Year 1/projects/9 FNC/fnc-1-ensemble/ensemble/XiaoxuanWang.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    352\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    353\u001b[0m         \u001b[0mXs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mys\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 354\u001b[0;31m         \u001b[0mprd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmlpc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    355\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mprd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/james/anaconda/lib/python3.5/site-packages/sklearn/neural_network/multilayer_perceptron.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    932\u001b[0m         \"\"\"\n\u001b[1;32m    933\u001b[0m         \u001b[0mcheck_is_fitted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"coefs_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 934\u001b[0;31m         \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    935\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    936\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_outputs_\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/james/anaconda/lib/python3.5/site-packages/sklearn/neural_network/multilayer_perceptron.py\u001b[0m in \u001b[0;36m_predict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    656\u001b[0m             \u001b[0mThe\u001b[0m \u001b[0mdecision\u001b[0m \u001b[0mfunction\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0msamples\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0meach\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    657\u001b[0m         \"\"\"\n\u001b[0;32m--> 658\u001b[0;31m         \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'csr'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'csc'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'coo'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    659\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    660\u001b[0m         \u001b[0;31m# Make sure self.hidden_layer_sizes is a list\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/james/anaconda/lib/python3.5/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[0;34m(array, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)\u001b[0m\n\u001b[1;32m    405\u001b[0m                              % (array.ndim, estimator_name))\n\u001b[1;32m    406\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mforce_all_finite\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 407\u001b[0;31m             \u001b[0m_assert_all_finite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    408\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    409\u001b[0m     \u001b[0mshape_repr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_shape_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/james/anaconda/lib/python3.5/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36m_assert_all_finite\u001b[0;34m(X)\u001b[0m\n\u001b[1;32m     56\u001b[0m             and not np.isfinite(X).all()):\n\u001b[1;32m     57\u001b[0m         raise ValueError(\"Input contains NaN, infinity\"\n\u001b[0;32m---> 58\u001b[0;31m                          \" or a value too large for %r.\" % X.dtype)\n\u001b[0m\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Input contains NaN, infinity or a value too large for dtype('float64')."
     ]
    }
   ],
   "source": [
    "import csv\n",
    "import sys\n",
    "import numpy as np\n",
    "from scipy import spatial\n",
    "\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from tqdm import tqdm\n",
    "\n",
    "from ensemble.GiorgosMyrianthous import GiorgosMyrianthous\n",
    "from ensemble.JiashuPu import JiashuPu\n",
    "from ensemble.FNCBaseLine import FNCBaseLine\n",
    "from ensemble.Master import Master\n",
    "from ensemble.MingjieChen import MingjieChen\n",
    "from ensemble.XiaoxuanWang import XiaoxuanWang\n",
    "from feature_engineering import refuting_features, polarity_features, hand_features, gen_or_load_feats\n",
    "from feature_engineering import word_overlap_features\n",
    "from upperbound import compute_ub\n",
    "from utils.dataset import DataSet\n",
    "from utils.generate_test_splits import kfold_split, get_stances_for_folds\n",
    "from utils.score import report_score, LABELS, score_submission\n",
    "import pickle\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    d = DataSet()\n",
    "    folds,hold_out = kfold_split(d,n_folds=2)\n",
    "    fold_stances, hold_out_stances = get_stances_for_folds(d,folds,hold_out)\n",
    "\n",
    "    Xs = dict()\n",
    "    ys = dict()\n",
    "\n",
    "    master_classifier = None\n",
    "\n",
    "    train = dict()\n",
    "    test = dict()\n",
    "\n",
    "    ids = list(range(len(folds)))\n",
    "    all_folds = np.hstack(tuple([fold_stances[i] for i in ids]))\n",
    "\n",
    "    for fold in fold_stances:\n",
    "        ids = list(range(len(folds)))\n",
    "        del ids[fold]\n",
    "\n",
    "        train[fold] = np.hstack(tuple([fold_stances[i] for i in ids]))\n",
    "        test[fold] = fold_stances[fold]\n",
    "\n",
    "    #\n",
    "    slave_classifiers = [FNCBaseLine,XiaoxuanWang,JiashuPu,GiorgosMyrianthous,MingjieChen]\n",
    "\n",
    "    slv_predicted = dict()\n",
    "    master_train = dict()\n",
    "\n",
    "    import os\n",
    "\n",
    "    if not os.path.isfile(\"features/master_train.pickle\"):\n",
    "        for fold in tqdm(fold_stances):\n",
    "            slv_predicted[fold] = []\n",
    "            master_train[fold] = []\n",
    "            for slv in tqdm(slave_classifiers):\n",
    "                print(\"Create classifier\" + str(slv))\n",
    "                cls = slv(d,all_folds)\n",
    "\n",
    "                print(\"Preload training data\" + str(type(cls)))\n",
    "                cls.preload_features(d.stances)\n",
    "\n",
    "                print(\"Train on fold \" + str(fold) + \" - \" + str(type(cls)))\n",
    "                cls.train(train[fold])\n",
    "\n",
    "                slv_predicted[fold].append([LABELS.index(p) for p in cls.predict(test[fold])])\n",
    "                del cls\n",
    "\n",
    "            master_train[fold].extend(zip(test[fold], *slv_predicted[fold]))\n",
    "\n",
    "        pickle.dump(master_train, open(\"features/master_train.pickle\",\"wb+\"))\n",
    "    else:\n",
    "        master_train = pickle.load(open(\"features/master_train.pickle\",\"rb\"))\n",
    "\n",
    "    slaves = []\n",
    "    if not os.path.isfile(\"features/slaves.pickle\"):\n",
    "        for slv in tqdm(slave_classifiers):\n",
    "            print(\"Training classifier\" + str(type(slv)))\n",
    "            cls = slv(d,all_folds)\n",
    "            cls.preload_features(d.stances)\n",
    "            cls.train(all_folds)\n",
    "            slaves.append(cls)\n",
    "            cls.delete_big_files()\n",
    "        pickle.dump(slaves, open(\"features/slaves.pickle\",\"wb+\"))\n",
    "    else:\n",
    "        slaves = pickle.load(open(\"features/slaves.pickle\",\"rb\"))\n",
    "\n",
    "\n",
    "    for slave in slaves:\n",
    "        print(\"Loading features for slave \" + str(type(slave)))\n",
    "        slave.preload_features(d.stances)\n",
    "        slave.load_w2v()\n",
    "\n",
    "\n",
    "    print(\"UPPER BOUND:::\")\n",
    "    compute_ub(slaves,hold_out_stances)\n",
    "\n",
    "    mdata = []\n",
    "    for fold in fold_stances:\n",
    "        mdata.extend(master_train[fold])\n",
    "    master = Master(d,mdata)\n",
    "    master.preload_features(d.stances)\n",
    "    master.fit(mdata)\n",
    "\n",
    "    slv_predicted_holdout = []\n",
    "    for slave in slaves:\n",
    "        slv_predicted_holdout.append([LABELS.index(p) for p in slave.predict(hold_out_stances)])\n",
    "\n",
    "    final_predictions = master.predict(zip(hold_out_stances,*slv_predicted_holdout))\n",
    "    report_score(master.xys(hold_out_stances)[1],final_predictions)\n",
    "\n",
    "    test_dataset = DataSet(\"test\")\n",
    "    d.articles.update(test_dataset.articles)\n",
    "\n",
    "    for stance in test_dataset.stances:\n",
    "        stance['Stance ID'] += len(d.stances)\n",
    "\n",
    "\n",
    "    slv_predicted_test = []\n",
    "    for slave in slaves:\n",
    "        slave.dataset.articles.update(test_dataset.articles)\n",
    "        slave.prepare_final(d,test_dataset,all_folds)\n",
    "        slave.preload_features(test_dataset.stances,\"test.\")\n",
    "        slv_predicted_test.append([LABELS.index(p) for p in slave.predict(test_dataset.stances)])\n",
    "\n",
    "    final_predictions = master.predict(zip(test_dataset.stances,*slv_predicted_test))\n",
    "\n",
    "    for label,stance in zip(final_predictions,test_dataset.stances):\n",
    "        stance['Stance'] = label\n",
    "        del stance['Stance ID']\n",
    "\n",
    "    f = open('submission.csv', 'wb')\n",
    "    w = csv.DictWriter(f, [\"Headline\",\"Body ID\", \"Stance\"])\n",
    "    w.writerows(test_dataset)\n",
    "    f.close()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 25413/25413 [01:42<00:00, 248.93it/s]\n"
     ]
    }
   ],
   "source": [
    "slave.preload_features(test_dataset.stances,\"test.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65654\n",
      "65894\n",
      "65997\n",
      "66152\n",
      "66376\n",
      "66567\n",
      "68205\n",
      "68387\n",
      "68630\n",
      "68736\n",
      "68802\n",
      "69135\n",
      "69829\n",
      "69942\n",
      "70248\n",
      "70430\n",
      "70434\n",
      "71592\n",
      "71668\n",
      "71686\n",
      "72077\n",
      "72521\n",
      "72600\n",
      "73156\n",
      "73305\n",
      "73367\n",
      "73753\n",
      "73818\n",
      "73983\n",
      "74238\n",
      "74418\n",
      "74824\n",
      "75003\n",
      "50023\n",
      "50110\n",
      "50409\n",
      "50437\n",
      "50713\n",
      "51000\n",
      "51179\n",
      "51241\n",
      "51329\n",
      "52218\n",
      "52362\n",
      "52709\n",
      "52730\n",
      "53038\n",
      "53632\n",
      "53811\n",
      "53943\n",
      "54313\n",
      "54544\n",
      "55065\n",
      "55584\n",
      "55774\n",
      "56541\n",
      "56622\n",
      "56641\n",
      "56676\n",
      "56954\n",
      "57058\n",
      "57202\n",
      "57410\n",
      "57809\n",
      "58631\n",
      "58650\n",
      "58964\n",
      "59114\n",
      "59177\n",
      "59194\n",
      "59358\n",
      "59399\n",
      "59421\n",
      "59519\n",
      "59537\n",
      "59568\n",
      "59786\n",
      "60054\n",
      "60155\n",
      "60377\n",
      "60393\n",
      "60860\n",
      "61140\n",
      "61271\n",
      "61983\n",
      "62967\n",
      "63121\n",
      "63397\n",
      "63730\n",
      "63830\n",
      "63999\n",
      "64112\n",
      "64269\n",
      "64521\n",
      "64538\n",
      "65090\n"
     ]
    }
   ],
   "source": [
    "for k in slave.tfidfs.keys():\n",
    "    if False == np.isfinite(slave.tfidfs[k]):\n",
    "        print(k)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}