BOW-SK-Learn-N_grams_ANN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TODO: Figure out ravel() 1d array problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from nltk import word_tokenize, sent_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from __future__ import division\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "labels = pd.read_csv('stocknews//labels.csv', header=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "http://stackoverflow.com/questions/28382735/python-pandas-does-not-read-the-first-row-of-csv-file\n",
    "\n",
    "`pd.read_csv` was cutting off the first row of labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1989, 1)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Confirm size of labels to make sure data loaded correctly\n",
    "labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "str_tokens_sw = pd.read_csv('stocknews//tokens_str_all.csv', dtype=str, keep_default_na=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Top1</th>\n",
       "      <th>Top2</th>\n",
       "      <th>Top3</th>\n",
       "      <th>Top4</th>\n",
       "      <th>Top5</th>\n",
       "      <th>Top6</th>\n",
       "      <th>Top7</th>\n",
       "      <th>Top8</th>\n",
       "      <th>Top9</th>\n",
       "      <th>Top10</th>\n",
       "      <th>...</th>\n",
       "      <th>Top16</th>\n",
       "      <th>Top17</th>\n",
       "      <th>Top18</th>\n",
       "      <th>Top19</th>\n",
       "      <th>Top20</th>\n",
       "      <th>Top21</th>\n",
       "      <th>Top22</th>\n",
       "      <th>Top23</th>\n",
       "      <th>Top24</th>\n",
       "      <th>Top25</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>georgia downs two russian warplanes as countri...</td>\n",
       "      <td>breaking musharraf to be impeached</td>\n",
       "      <td>russia today columns of troops roll into south...</td>\n",
       "      <td>russian tanks are moving towards the capital o...</td>\n",
       "      <td>afghan children raped with impunity official s...</td>\n",
       "      <td>150 russian tanks have entered south ossetia w...</td>\n",
       "      <td>breaking georgia invades south ossetia russia ...</td>\n",
       "      <td>the enemy combatent trials are nothing but sha...</td>\n",
       "      <td>georgian troops retreat from osettain capital ...</td>\n",
       "      <td>did the prep georgia for war with russia</td>\n",
       "      <td>...</td>\n",
       "      <td>georgia invades south ossetia if russia gets i...</td>\n",
       "      <td>al qaeda faces islamist backlash</td>\n",
       "      <td>condoleezza rice the us would not act to preve...</td>\n",
       "      <td>this is busy day the european union has approv...</td>\n",
       "      <td>georgia will withdraw 000 soldiers from iraq t...</td>\n",
       "      <td>why the pentagon thinks attacking iran is bad ...</td>\n",
       "      <td>caucasus in crisis georgia invades south ossetia</td>\n",
       "      <td>indian shoe manufactory and again in series of...</td>\n",
       "      <td>visitors suffering from mental illnesses banne...</td>\n",
       "      <td>no help for mexico kidnapping surge</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>why wont america and nato help us if they wont...</td>\n",
       "      <td>bush puts foot down on georgian conflict</td>\n",
       "      <td>jewish georgian minister thanks to israeli tra...</td>\n",
       "      <td>georgian army flees in disarray as russians ad...</td>\n",
       "      <td>olympic opening ceremony fireworks faked</td>\n",
       "      <td>what were the mossad with fraudulent new zeala...</td>\n",
       "      <td>russia angered by israeli military sale to geo...</td>\n",
       "      <td>an american citizen living in ossetia blames a...</td>\n",
       "      <td>welcome to world war iv now in high definition</td>\n",
       "      <td>georgia move mistake of monumental proportions</td>\n",
       "      <td>...</td>\n",
       "      <td>israel and the us behind the georgian aggression</td>\n",
       "      <td>do not believe tv neither russian nor georgian...</td>\n",
       "      <td>riots are still going on in montreal canada be...</td>\n",
       "      <td>china to overtake us as largest manufacturer</td>\n",
       "      <td>war in south ossetia pics</td>\n",
       "      <td>israeli physicians group condemns state torture</td>\n",
       "      <td>russia has just beaten the united states over ...</td>\n",
       "      <td>perhaps the question about the georgia russia ...</td>\n",
       "      <td>russia is so much better at war</td>\n",
       "      <td>so this is what it come to trading sex for food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>remember that adorable year old who sang at th...</td>\n",
       "      <td>russia ends georgia operation</td>\n",
       "      <td>if we had no sexual harassment we would have n...</td>\n",
       "      <td>al qa eda is losing support in iraq because of...</td>\n",
       "      <td>ceasefire in georgia putin outmaneuvers the west</td>\n",
       "      <td>why microsoft and intel tried to kill the xo 1...</td>\n",
       "      <td>stratfor the russo georgian war and the balanc...</td>\n",
       "      <td>trying to get sense of this whole georgia russ...</td>\n",
       "      <td>the us military was surprised by the timing an...</td>\n",
       "      <td>beats war drum as iran dumps the dollar</td>\n",
       "      <td>...</td>\n",
       "      <td>troops still in georgia did you know they were...</td>\n",
       "      <td>why russias response to georgia was right</td>\n",
       "      <td>gorbachev accuses of making serious blunder in...</td>\n",
       "      <td>russia georgia and nato cold war two</td>\n",
       "      <td>remember that adorable 62 year old who led you...</td>\n",
       "      <td>war in georgia the israeli connection</td>\n",
       "      <td>all signs point to the us encouraging georgia ...</td>\n",
       "      <td>christopher king argues that the us and nato a...</td>\n",
       "      <td>america the new mexico</td>\n",
       "      <td>bbc news asia pacific extinction by man not cl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>refuses israel weapons to attack iran report</td>\n",
       "      <td>when the president ordered to attack tskhinval...</td>\n",
       "      <td>israel clears troops who killed reuters cameraman</td>\n",
       "      <td>britain policy of being tough on drugs is poin...</td>\n",
       "      <td>body of 14 year old found in trunk latest rans...</td>\n",
       "      <td>china has moved 10 million quake survivors int...</td>\n",
       "      <td>bush announces operation get all up in russia ...</td>\n",
       "      <td>russian forces sink georgian ships</td>\n",
       "      <td>the commander of navy air reconnaissance squad...</td>\n",
       "      <td>92 of cnn readers russia actions in georgia ju...</td>\n",
       "      <td>...</td>\n",
       "      <td>elephants extinct by 2020</td>\n",
       "      <td>us humanitarian missions soon in georgia if ru...</td>\n",
       "      <td>georgia ddos came from us sources</td>\n",
       "      <td>russian convoy heads into georgia violating truce</td>\n",
       "      <td>israeli defence minister us against strike on ...</td>\n",
       "      <td>gorbachev we had no choice</td>\n",
       "      <td>witness russian forces head towards tbilisi in...</td>\n",
       "      <td>quarter of russians blame for conflict poll</td>\n",
       "      <td>georgian president says us military will take ...</td>\n",
       "      <td>2006 nobel laureate aleksander solzhenitsyn ac...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>all the experts admit that we should legalise ...</td>\n",
       "      <td>war in south osetia 89 pictures made by russia...</td>\n",
       "      <td>swedish wrestler ara abrahamian throws away me...</td>\n",
       "      <td>russia exaggerated the death toll in south oss...</td>\n",
       "      <td>missile that killed inside pakistan may have b...</td>\n",
       "      <td>rushdie condemns random house refusal to publi...</td>\n",
       "      <td>poland and us agree to missle defense deal int...</td>\n",
       "      <td>will the russians conquer tblisi bet on it no ...</td>\n",
       "      <td>russia exaggerating south ossetian death toll ...</td>\n",
       "      <td>musharraf expected to resign rather than face ...</td>\n",
       "      <td>...</td>\n",
       "      <td>bank analyst forecast georgian crisis days early</td>\n",
       "      <td>georgia confict could set back russia us relat...</td>\n",
       "      <td>war in the caucasus is as much the product of ...</td>\n",
       "      <td>non media photos of south ossetia georgia conf...</td>\n",
       "      <td>georgian tv reporter shot by russian sniper du...</td>\n",
       "      <td>saudi arabia mother moves to block child marriage</td>\n",
       "      <td>taliban wages war on humanitarian aid workers</td>\n",
       "      <td>russia world can forget about georgia territor...</td>\n",
       "      <td>darfur rebels accuse sudan of mounting major a...</td>\n",
       "      <td>philippines peace advocate say muslims need as...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Top1  \\\n",
       "0  georgia downs two russian warplanes as countri...   \n",
       "1  why wont america and nato help us if they wont...   \n",
       "2  remember that adorable year old who sang at th...   \n",
       "3       refuses israel weapons to attack iran report   \n",
       "4  all the experts admit that we should legalise ...   \n",
       "\n",
       "                                                Top2  \\\n",
       "0                 breaking musharraf to be impeached   \n",
       "1           bush puts foot down on georgian conflict   \n",
       "2                      russia ends georgia operation   \n",
       "3  when the president ordered to attack tskhinval...   \n",
       "4  war in south osetia 89 pictures made by russia...   \n",
       "\n",
       "                                                Top3  \\\n",
       "0  russia today columns of troops roll into south...   \n",
       "1  jewish georgian minister thanks to israeli tra...   \n",
       "2  if we had no sexual harassment we would have n...   \n",
       "3  israel clears troops who killed reuters cameraman   \n",
       "4  swedish wrestler ara abrahamian throws away me...   \n",
       "\n",
       "                                                Top4  \\\n",
       "0  russian tanks are moving towards the capital o...   \n",
       "1  georgian army flees in disarray as russians ad...   \n",
       "2  al qa eda is losing support in iraq because of...   \n",
       "3  britain policy of being tough on drugs is poin...   \n",
       "4  russia exaggerated the death toll in south oss...   \n",
       "\n",
       "                                                Top5  \\\n",
       "0  afghan children raped with impunity official s...   \n",
       "1           olympic opening ceremony fireworks faked   \n",
       "2   ceasefire in georgia putin outmaneuvers the west   \n",
       "3  body of 14 year old found in trunk latest rans...   \n",
       "4  missile that killed inside pakistan may have b...   \n",
       "\n",
       "                                                Top6  \\\n",
       "0  150 russian tanks have entered south ossetia w...   \n",
       "1  what were the mossad with fraudulent new zeala...   \n",
       "2  why microsoft and intel tried to kill the xo 1...   \n",
       "3  china has moved 10 million quake survivors int...   \n",
       "4  rushdie condemns random house refusal to publi...   \n",
       "\n",
       "                                                Top7  \\\n",
       "0  breaking georgia invades south ossetia russia ...   \n",
       "1  russia angered by israeli military sale to geo...   \n",
       "2  stratfor the russo georgian war and the balanc...   \n",
       "3  bush announces operation get all up in russia ...   \n",
       "4  poland and us agree to missle defense deal int...   \n",
       "\n",
       "                                                Top8  \\\n",
       "0  the enemy combatent trials are nothing but sha...   \n",
       "1  an american citizen living in ossetia blames a...   \n",
       "2  trying to get sense of this whole georgia russ...   \n",
       "3                 russian forces sink georgian ships   \n",
       "4  will the russians conquer tblisi bet on it no ...   \n",
       "\n",
       "                                                Top9  \\\n",
       "0  georgian troops retreat from osettain capital ...   \n",
       "1     welcome to world war iv now in high definition   \n",
       "2  the us military was surprised by the timing an...   \n",
       "3  the commander of navy air reconnaissance squad...   \n",
       "4  russia exaggerating south ossetian death toll ...   \n",
       "\n",
       "                                               Top10  \\\n",
       "0           did the prep georgia for war with russia   \n",
       "1     georgia move mistake of monumental proportions   \n",
       "2            beats war drum as iran dumps the dollar   \n",
       "3  92 of cnn readers russia actions in georgia ju...   \n",
       "4  musharraf expected to resign rather than face ...   \n",
       "\n",
       "                         ...                          \\\n",
       "0                        ...                           \n",
       "1                        ...                           \n",
       "2                        ...                           \n",
       "3                        ...                           \n",
       "4                        ...                           \n",
       "\n",
       "                                               Top16  \\\n",
       "0  georgia invades south ossetia if russia gets i...   \n",
       "1   israel and the us behind the georgian aggression   \n",
       "2  troops still in georgia did you know they were...   \n",
       "3                          elephants extinct by 2020   \n",
       "4   bank analyst forecast georgian crisis days early   \n",
       "\n",
       "                                               Top17  \\\n",
       "0                   al qaeda faces islamist backlash   \n",
       "1  do not believe tv neither russian nor georgian...   \n",
       "2          why russias response to georgia was right   \n",
       "3  us humanitarian missions soon in georgia if ru...   \n",
       "4  georgia confict could set back russia us relat...   \n",
       "\n",
       "                                               Top18  \\\n",
       "0  condoleezza rice the us would not act to preve...   \n",
       "1  riots are still going on in montreal canada be...   \n",
       "2  gorbachev accuses of making serious blunder in...   \n",
       "3                  georgia ddos came from us sources   \n",
       "4  war in the caucasus is as much the product of ...   \n",
       "\n",
       "                                               Top19  \\\n",
       "0  this is busy day the european union has approv...   \n",
       "1       china to overtake us as largest manufacturer   \n",
       "2               russia georgia and nato cold war two   \n",
       "3  russian convoy heads into georgia violating truce   \n",
       "4  non media photos of south ossetia georgia conf...   \n",
       "\n",
       "                                               Top20  \\\n",
       "0  georgia will withdraw 000 soldiers from iraq t...   \n",
       "1                          war in south ossetia pics   \n",
       "2  remember that adorable 62 year old who led you...   \n",
       "3  israeli defence minister us against strike on ...   \n",
       "4  georgian tv reporter shot by russian sniper du...   \n",
       "\n",
       "                                               Top21  \\\n",
       "0  why the pentagon thinks attacking iran is bad ...   \n",
       "1    israeli physicians group condemns state torture   \n",
       "2              war in georgia the israeli connection   \n",
       "3                         gorbachev we had no choice   \n",
       "4  saudi arabia mother moves to block child marriage   \n",
       "\n",
       "                                               Top22  \\\n",
       "0   caucasus in crisis georgia invades south ossetia   \n",
       "1  russia has just beaten the united states over ...   \n",
       "2  all signs point to the us encouraging georgia ...   \n",
       "3  witness russian forces head towards tbilisi in...   \n",
       "4      taliban wages war on humanitarian aid workers   \n",
       "\n",
       "                                               Top23  \\\n",
       "0  indian shoe manufactory and again in series of...   \n",
       "1  perhaps the question about the georgia russia ...   \n",
       "2  christopher king argues that the us and nato a...   \n",
       "3        quarter of russians blame for conflict poll   \n",
       "4  russia world can forget about georgia territor...   \n",
       "\n",
       "                                               Top24  \\\n",
       "0  visitors suffering from mental illnesses banne...   \n",
       "1                    russia is so much better at war   \n",
       "2                             america the new mexico   \n",
       "3  georgian president says us military will take ...   \n",
       "4  darfur rebels accuse sudan of mounting major a...   \n",
       "\n",
       "                                               Top25  \n",
       "0                no help for mexico kidnapping surge  \n",
       "1    so this is what it come to trading sex for food  \n",
       "2  bbc news asia pacific extinction by man not cl...  \n",
       "3  2006 nobel laureate aleksander solzhenitsyn ac...  \n",
       "4  philippines peace advocate say muslims need as...  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str_tokens_sw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1989, 25)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str_tokens_sw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1989 entries, 0 to 1988\n",
      "Data columns (total 25 columns):\n",
      "Top1     1989 non-null object\n",
      "Top2     1989 non-null object\n",
      "Top3     1989 non-null object\n",
      "Top4     1989 non-null object\n",
      "Top5     1989 non-null object\n",
      "Top6     1989 non-null object\n",
      "Top7     1989 non-null object\n",
      "Top8     1989 non-null object\n",
      "Top9     1989 non-null object\n",
      "Top10    1989 non-null object\n",
      "Top11    1989 non-null object\n",
      "Top12    1989 non-null object\n",
      "Top13    1989 non-null object\n",
      "Top14    1989 non-null object\n",
      "Top15    1989 non-null object\n",
      "Top16    1989 non-null object\n",
      "Top17    1989 non-null object\n",
      "Top18    1989 non-null object\n",
      "Top19    1989 non-null object\n",
      "Top20    1989 non-null object\n",
      "Top21    1989 non-null object\n",
      "Top22    1989 non-null object\n",
      "Top23    1989 non-null object\n",
      "Top24    1989 non-null object\n",
      "Top25    1989 non-null object\n",
      "dtypes: object(25)\n",
      "memory usage: 388.5+ KB\n"
     ]
    }
   ],
   "source": [
    "str_tokens_sw.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Top1</th>\n",
       "      <th>Top2</th>\n",
       "      <th>Top3</th>\n",
       "      <th>Top4</th>\n",
       "      <th>Top5</th>\n",
       "      <th>Top6</th>\n",
       "      <th>Top7</th>\n",
       "      <th>Top8</th>\n",
       "      <th>Top9</th>\n",
       "      <th>Top10</th>\n",
       "      <th>...</th>\n",
       "      <th>Top16</th>\n",
       "      <th>Top17</th>\n",
       "      <th>Top18</th>\n",
       "      <th>Top19</th>\n",
       "      <th>Top20</th>\n",
       "      <th>Top21</th>\n",
       "      <th>Top22</th>\n",
       "      <th>Top23</th>\n",
       "      <th>Top24</th>\n",
       "      <th>Top25</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>...</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>1988</td>\n",
       "      <td>1989</td>\n",
       "      <td>1988</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1988</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>...</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1989</td>\n",
       "      <td>1987</td>\n",
       "      <td>1987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>monsanto uprooted germany bans cultivation of ...</td>\n",
       "      <td>hunt for skilled labor germany woos portugal l...</td>\n",
       "      <td>putin who gave nato right to kill gaddafi</td>\n",
       "      <td>french people back the burqa ban by margin of ...</td>\n",
       "      <td>belgium to ban islamic veils in public making ...</td>\n",
       "      <td>from the delhi police why women deserve to be ...</td>\n",
       "      <td>it official china is the world biggest polluter</td>\n",
       "      <td>600 on hunger strike and the world doesnt even...</td>\n",
       "      <td>hundreds gathered on wednesday in front of the...</td>\n",
       "      <td>ukrainian customs block 10 tons of humanitaria...</td>\n",
       "      <td>...</td>\n",
       "      <td>poll 70 of israelis say israel should accept u...</td>\n",
       "      <td>girls only allowed to access facebook if they ...</td>\n",
       "      <td>in the painful tumult of daily protests an ent...</td>\n",
       "      <td>nsa chief may lose cyber command powers</td>\n",
       "      <td>senate committee backs bill that would allow n...</td>\n",
       "      <td>green pope pope to write encyclical doctrinal ...</td>\n",
       "      <td>three british men have claimed they were beate...</td>\n",
       "      <td>britain to send military advisers to aid libya...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     Top1  \\\n",
       "count                                                1989   \n",
       "unique                                               1988   \n",
       "top     monsanto uprooted germany bans cultivation of ...   \n",
       "freq                                                    2   \n",
       "\n",
       "                                                     Top2  \\\n",
       "count                                                1989   \n",
       "unique                                               1989   \n",
       "top     hunt for skilled labor germany woos portugal l...   \n",
       "freq                                                    1   \n",
       "\n",
       "                                             Top3  \\\n",
       "count                                        1989   \n",
       "unique                                       1988   \n",
       "top     putin who gave nato right to kill gaddafi   \n",
       "freq                                            2   \n",
       "\n",
       "                                                     Top4  \\\n",
       "count                                                1989   \n",
       "unique                                               1989   \n",
       "top     french people back the burqa ban by margin of ...   \n",
       "freq                                                    1   \n",
       "\n",
       "                                                     Top5  \\\n",
       "count                                                1989   \n",
       "unique                                               1989   \n",
       "top     belgium to ban islamic veils in public making ...   \n",
       "freq                                                    1   \n",
       "\n",
       "                                                     Top6  \\\n",
       "count                                                1989   \n",
       "unique                                               1988   \n",
       "top     from the delhi police why women deserve to be ...   \n",
       "freq                                                    2   \n",
       "\n",
       "                                                   Top7  \\\n",
       "count                                              1989   \n",
       "unique                                             1989   \n",
       "top     it official china is the world biggest polluter   \n",
       "freq                                                  1   \n",
       "\n",
       "                                                     Top8  \\\n",
       "count                                                1989   \n",
       "unique                                               1989   \n",
       "top     600 on hunger strike and the world doesnt even...   \n",
       "freq                                                    1   \n",
       "\n",
       "                                                     Top9  \\\n",
       "count                                                1989   \n",
       "unique                                               1989   \n",
       "top     hundreds gathered on wednesday in front of the...   \n",
       "freq                                                    1   \n",
       "\n",
       "                                                    Top10  ...   \\\n",
       "count                                                1989  ...    \n",
       "unique                                               1989  ...    \n",
       "top     ukrainian customs block 10 tons of humanitaria...  ...    \n",
       "freq                                                    1  ...    \n",
       "\n",
       "                                                    Top16  \\\n",
       "count                                                1989   \n",
       "unique                                               1989   \n",
       "top     poll 70 of israelis say israel should accept u...   \n",
       "freq                                                    1   \n",
       "\n",
       "                                                    Top17  \\\n",
       "count                                                1989   \n",
       "unique                                               1989   \n",
       "top     girls only allowed to access facebook if they ...   \n",
       "freq                                                    1   \n",
       "\n",
       "                                                    Top18  \\\n",
       "count                                                1989   \n",
       "unique                                               1989   \n",
       "top     in the painful tumult of daily protests an ent...   \n",
       "freq                                                    1   \n",
       "\n",
       "                                          Top19  \\\n",
       "count                                      1989   \n",
       "unique                                     1989   \n",
       "top     nsa chief may lose cyber command powers   \n",
       "freq                                          1   \n",
       "\n",
       "                                                    Top20  \\\n",
       "count                                                1989   \n",
       "unique                                               1989   \n",
       "top     senate committee backs bill that would allow n...   \n",
       "freq                                                    1   \n",
       "\n",
       "                                                    Top21  \\\n",
       "count                                                1989   \n",
       "unique                                               1989   \n",
       "top     green pope pope to write encyclical doctrinal ...   \n",
       "freq                                                    1   \n",
       "\n",
       "                                                    Top22  \\\n",
       "count                                                1989   \n",
       "unique                                               1989   \n",
       "top     three british men have claimed they were beate...   \n",
       "freq                                                    1   \n",
       "\n",
       "                                                    Top23 Top24 Top25  \n",
       "count                                                1989  1989  1989  \n",
       "unique                                               1989  1987  1987  \n",
       "top     britain to send military advisers to aid libya...              \n",
       "freq                                                    1     3     3  \n",
       "\n",
       "[4 rows x 25 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str_tokens_sw.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.set_random_seed(123)\n",
    "np.random.seed(123)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace_num(element):\n",
    "    return ' '.join([('numero' if k.isdigit() else k) for k in element.split()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "http://stackoverflow.com/questions/6905636/python-conditional-list-joins"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Instead of a digit, use `'NUMERO'` as it is resistant to stemming/lemmatizing.  It's possible that headlines with numbers might contain some information.  This will prevent preprocessing from discarding the information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "str_tokens_sw['merged'] = str_tokens_sw.iloc[:, 0:].apply(lambda x: ' '.join(x.dropna().values.tolist()), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# type(str_tokens['merged'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "str_tokens_sw['merged'] = str_tokens_sw['merged'].apply(replace_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.stem.porter import PorterStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "pstem = PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stm(element):\n",
    "    return ' '.join([pstem.stem(k) for k in element.split()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "str_tokens_sw['merged'] = str_tokens_sw['merged'].apply(stm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       georgia down two russian warplan as countri mo...\n",
       "1       whi wont america and nato help us if they wont...\n",
       "2       rememb that ador year old who sang at the open...\n",
       "3       refus israel weapon to attack iran report when...\n",
       "4       all the expert admit that we should legalis dr...\n",
       "5       mom of miss gay man too bad he not numero year...\n",
       "6       in an afghan prison the major of femal prison ...\n",
       "7       man arrest and lock up for five hour after tak...\n",
       "8       two elderli chines women have been sentenc to ...\n",
       "9       british resid held in guantanamo bay win legal...\n",
       "10      syria say it readi to put russian missil syste...\n",
       "11      korea kim die in numero replac by lookalik say...\n",
       "12      north korea halt denuclearis after us fail to ...\n",
       "13      photo of numero year old iraqi suicid bomber w...\n",
       "14      militari help for georgia is declar of war say...\n",
       "15      russian prime minist vladimir putin ha accus t...\n",
       "16      girl film isra soldier brutal sinc then her fa...\n",
       "17      poland legal of marijuana two majour step in t...\n",
       "18      secur guard beat man at soccer footbal game fa...\n",
       "19      in jordan the honor kill that are not women ar...\n",
       "20      french judg ha order two branch of scientologi...\n",
       "21      love the world thai prime minist samak sundara...\n",
       "22      dutch to ban burka when pick children up from ...\n",
       "23      american intellig confirm that the latest mili...\n",
       "24      opec finish saudi arabia say later putin may s...\n",
       "25      did saakashvili lie the west begin to doubt ge...\n",
       "26      pakistani tribesmen repres half million peopl ...\n",
       "27      attack on us embassi in yemen leav numero dead...\n",
       "28      numero babi sick milk scandal spread afghan fa...\n",
       "29      brit take stand ban short sell of financi stoc...\n",
       "                              ...                        \n",
       "1959    linkedin just reveal that the numero breach th...\n",
       "1960    it keep happen rape survivor say protest again...\n",
       "1961    colombia ha fire more than numero polic offic ...\n",
       "1962    adida shoe manufactur return to germani fulli ...\n",
       "1963    greek news site are report that aristotl tomb ...\n",
       "1964    thailand ha shut down numero popular dive site...\n",
       "1965    everi refer to the great barrier reef remov fr...\n",
       "1966    forti tiger cub bodi have been found in freeze...\n",
       "1967    germani recogn the armenian genocid media is i...\n",
       "1968    seven year old boy who went miss in bear inhab...\n",
       "1969    nearli numero of qatar popul live in labour ca...\n",
       "1970    five men in india have been convict for the ga...\n",
       "1971    un remov saudi arabia from human right blackli...\n",
       "1972    vandal destroy numero yr old aborigin artwork ...\n",
       "1973    chief admit he remov saudi arabia from child k...\n",
       "1974    dutch woman who alleg she wa rape after be dru...\n",
       "1975    gay chines man sue mental hospit for tri to cu...\n",
       "1976    isi twitter account have been hijack with gay ...\n",
       "1977    decriminalis call as war on drug fail top publ...\n",
       "1978    mp jo cox dead after shoot attack saudi arabia...\n",
       "1979    stagger numero percent of venezuelan say they ...\n",
       "1980    an australian athlet who ha compet in six para...\n",
       "1981    german govern agre to ban frack indefinit teen...\n",
       "1982    today the unit kingdom decid whether to remain...\n",
       "1983    david cameron to resign as pm after eu referen...\n",
       "1984    barclay and rb share suspend from trade after ...\n",
       "1985    numero scientist to australia if you want to s...\n",
       "1986    explos at airport in istanbul yemeni former pr...\n",
       "1987    jamaica propos marijuana dispens for tourist a...\n",
       "1988    numero year old woman in mexico citi final rec...\n",
       "Name: merged, Length: 1989, dtype: object"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str_tokens_sw['merged']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split labels and tokens into 4 folds\n",
    "Fold 1 = [:499]\n",
    "Fold 2 = [499:996]\n",
    "Fold 3 = [996:1493]\n",
    "Fold 4 = [1493:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split data into train and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_text = str_tokens_sw.merged[0:1493] # train features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_text = str_tokens_sw.merged[1493:] # test features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels = labels[0:1493].values # train labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_labels = labels[1493:].values; # test labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer( max_features=5000, ngram_range = ( 1, 4 ), sublinear_tf = True, max_df=0.66 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Only need text, not labels\n",
    "train_x = vectorizer.fit_transform( train_text )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_x = vectorizer.transform( test_text )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1493, 5000)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(496, 5000)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1493, 1)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(496, 1)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_labels.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensorflow ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X = tf.cast(X, dtype=tf.float64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.08"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 149"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_0 = tf.placeholder(tf.float64, [None, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float64, [None, 5000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1st layer\n",
    "W_1 = tf.Variable(tf.zeros([5000, 50]))\n",
    "b_1 = tf.Variable(tf.zeros([50]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "W_1 = tf.cast(W_1, dtype=tf.float64)\n",
    "b_1 = tf.cast(b_1, dtype=tf.float64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_1 = tf.matmul(X, W_1) + b_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 2nd layer\n",
    "W_2 = tf.Variable(tf.zeros([50,1]))\n",
    "b_2 = tf.Variable(tf.zeros([1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "W_2 = tf.cast(W_2, dtype=tf.float64)\n",
    "b_2 = tf.cast(b_2, dtype=tf.float64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_2 = tf.nn.softmax(tf.matmul(y_1, W_2) + b_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def model(X, W_1, b_1, W_2, b_2):\n",
    "    y_1 = tf.matmul(X,   W_1) + b_1\n",
    "    y_2 = tf.matmul(y_1, W_2) + b_2\n",
    "    return y_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_est = model(X, W_1, b_1, W_2, b_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_est, labels=y_0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predict_op = y_est"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, 0.0)\n",
      "(0, 0.0)\n",
      "(0, 0.0)\n",
      "(0, 0.0)\n",
      "(0, 0.0)\n",
      "(0, 0.0)\n",
      "(0, 0.0)\n",
      "(0, 0.0)\n",
      "(0, 0.0)\n",
      "(0, 0.0)\n",
      "(1, 0.0)\n",
      "(1, 0.0)\n",
      "(1, 0.0)\n",
      "(1, 0.0)\n",
      "(1, 0.0)\n",
      "(1, 0.0)\n",
      "(1, 0.0)\n",
      "(1, 0.0)\n",
      "(1, 0.0)\n",
      "(1, 0.0)\n",
      "(2, 0.0)\n",
      "(2, 0.0)\n",
      "(2, 0.0)\n",
      "(2, 0.0)\n",
      "(2, 0.0)\n",
      "(2, 0.0)\n",
      "(2, 0.0)\n",
      "(2, 0.0)\n",
      "(2, 0.0)\n",
      "(2, 0.0)\n",
      "(3, 0.0)\n",
      "(3, 0.0)\n",
      "(3, 0.0)\n",
      "(3, 0.0)\n",
      "(3, 0.0)\n",
      "(3, 0.0)\n",
      "(3, 0.0)\n",
      "(3, 0.0)\n",
      "(3, 0.0)\n",
      "(3, 0.0)\n",
      "(4, 0.0)\n",
      "(4, 0.0)\n",
      "(4, 0.0)\n",
      "(4, 0.0)\n",
      "(4, 0.0)\n",
      "(4, 0.0)\n",
      "(4, 0.0)\n",
      "(4, 0.0)\n",
      "(4, 0.0)\n",
      "(4, 0.0)\n",
      "(5, 0.0)\n",
      "(5, 0.0)\n",
      "(5, 0.0)\n",
      "(5, 0.0)\n",
      "(5, 0.0)\n",
      "(5, 0.0)\n",
      "(5, 0.0)\n",
      "(5, 0.0)\n",
      "(5, 0.0)\n",
      "(5, 0.0)\n",
      "(6, 0.0)\n",
      "(6, 0.0)\n",
      "(6, 0.0)\n",
      "(6, 0.0)\n",
      "(6, 0.0)\n",
      "(6, 0.0)\n",
      "(6, 0.0)\n",
      "(6, 0.0)\n",
      "(6, 0.0)\n",
      "(6, 0.0)\n",
      "(7, 0.0)\n",
      "(7, 0.0)\n",
      "(7, 0.0)\n",
      "(7, 0.0)\n",
      "(7, 0.0)\n",
      "(7, 0.0)\n",
      "(7, 0.0)\n",
      "(7, 0.0)\n",
      "(7, 0.0)\n",
      "(7, 0.0)\n",
      "(8, 0.0)\n",
      "(8, 0.0)\n",
      "(8, 0.0)\n",
      "(8, 0.0)\n",
      "(8, 0.0)\n",
      "(8, 0.0)\n",
      "(8, 0.0)\n",
      "(8, 0.0)\n",
      "(8, 0.0)\n",
      "(8, 0.0)\n",
      "(9, 0.0)\n",
      "(9, 0.0)\n",
      "(9, 0.0)\n",
      "(9, 0.0)\n",
      "(9, 0.0)\n",
      "(9, 0.0)\n",
      "(9, 0.0)\n",
      "(9, 0.0)\n",
      "(9, 0.0)\n",
      "(9, 0.0)\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    # initialize variables\n",
    "    init.run()\n",
    "    for i in range(10):\n",
    "        for start, end in zip(range(0, train_x.shape[0], batch_size), range(batch_size, train_x.shape[0]+1, batch_size)):\n",
    "            sess.run(train_op, feed_dict={X: train_x.A[start:end,:], y_0: train_labels[start:end]})\n",
    "            print(i, np.mean((test_labels) == sess.run(predict_op, feed_dict={X: test_x.A})))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}