07.1 Text Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trying out a simple learner\n",
    "\n",
    "Before we try to build our deep learning models, let's make sure we can learn something using a simple linear model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://www.crowdflower.com/wp-content/uploads/2016/07/text_emotion.csv\n",
      "3907584/4394791 [=========================>....] - ETA: 0s"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from keras.utils.data_utils import get_file\n",
    "import nb_utils\n",
    "\n",
    "emotion_csv = get_file('text_emotion.csv', \n",
    "                       'https://www.crowdflower.com/wp-content/uploads/2016/07/text_emotion.csv')\n",
    "emotion_df = pd.read_csv(emotion_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>author</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1956967341</td>\n",
       "      <td>empty</td>\n",
       "      <td>xoshayzers</td>\n",
       "      <td>@tiffanylue i know  i was listenin to bad habi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1956967666</td>\n",
       "      <td>sadness</td>\n",
       "      <td>wannamama</td>\n",
       "      <td>Layin n bed with a headache  ughhhh...waitin o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1956967696</td>\n",
       "      <td>sadness</td>\n",
       "      <td>coolfunky</td>\n",
       "      <td>Funeral ceremony...gloomy friday...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1956967789</td>\n",
       "      <td>enthusiasm</td>\n",
       "      <td>czareaquino</td>\n",
       "      <td>wants to hang out with friends SOON!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1956968416</td>\n",
       "      <td>neutral</td>\n",
       "      <td>xkilljoyx</td>\n",
       "      <td>@dannycastillo We want to trade with someone w...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     tweet_id   sentiment       author  \\\n",
       "0  1956967341       empty   xoshayzers   \n",
       "1  1956967666     sadness    wannamama   \n",
       "2  1956967696     sadness    coolfunky   \n",
       "3  1956967789  enthusiasm  czareaquino   \n",
       "4  1956968416     neutral    xkilljoyx   \n",
       "\n",
       "                                             content  \n",
       "0  @tiffanylue i know  i was listenin to bad habi...  \n",
       "1  Layin n bed with a headache  ughhhh...waitin o...  \n",
       "2                Funeral ceremony...gloomy friday...  \n",
       "3               wants to hang out with friends SOON!  \n",
       "4  @dannycastillo We want to trade with someone w...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emotion_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "neutral       8638\n",
       "worry         8459\n",
       "happiness     5209\n",
       "sadness       5165\n",
       "love          3842\n",
       "surprise      2187\n",
       "fun           1776\n",
       "relief        1526\n",
       "hate          1323\n",
       "empty          827\n",
       "enthusiasm     759\n",
       "boredom        179\n",
       "anger          110\n",
       "Name: sentiment, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emotion_df['sentiment'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "VOCAB_SIZE = 50000\n",
    "\n",
    "tfidf_vec = TfidfVectorizer(max_features=VOCAB_SIZE)\n",
    "label_encoder = LabelEncoder()\n",
    "\n",
    "X = tfidf_vec.fit_transform(emotion_df['content'])\n",
    "y = label_encoder.fit_transform(emotion_df['sentiment'])\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.28022727272727271"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bayes = MultinomialNB()\n",
    "bayes.fit(X_train, y_train)\n",
    "predictions = bayes.predict(X_test)\n",
    "precision_score(predictions, y_test, average='micro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('svm', 0.21863636363636363)\n",
      "('sgd', 0.32363636363636361)\n",
      "('random_forrest', 0.28227272727272729)\n"
     ]
    }
   ],
   "source": [
    "classifiers = {'sgd': SGDClassifier(loss='hinge'),\n",
    "               'svm': SVC(),\n",
    "               'random_forest': RandomForestClassifier()}\n",
    "\n",
    "for lbl, clf in classifiers.items():\n",
    "    clf.fit(X_train, y_train)\n",
    "    predictions = clf.predict(X_test)\n",
    "    print(lbl, precision_score(predictions, y_test, average='micro'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking what our model learned\n",
    "\n",
    "Our linear models appear to be learning something more powerful than \"pick the most popular category\".  We can take a quick look at which words they find the most correlated with each category before moving on to our neural network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.sparse import eye\n",
    "d = eye(len(tfidf_vec.vocabulary_))\n",
    "word_pred = bayes.predict_proba(d)\n",
    "inverse_vocab = {idx: word for word, idx in tfidf_vec.vocabulary_.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter, defaultdict\n",
    "by_cls = defaultdict(Counter)\n",
    "for word_idx, pred in enumerate(word_pred):\n",
    "    for class_idx, score in enumerate(pred):\n",
    "        cls = label_encoder.classes_[class_idx]\n",
    "        by_cls[cls][inverse_vocab[word_idx]] = score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "relief : finally relax mastered relief inspiration\n",
      "happiness : excited woohoo excellent yay wars\n",
      "enthusiasm : lena_distractia foolproofdiva attending krisswouldhowse tatt\n",
      "hate : hate hates suck fucking zomberellamcfox\n",
      "fun : xbox bamboozle sanctuary oldies toodaayy\n",
      "sadness : sad sadly cry cried miss\n",
      "empty : bethsybsb less_than_3 shakeyourjunk kimbermuffin _cheshire_cat_\n",
      "neutral : www painting souljaboytellem link frenchieb\n",
      "surprise : surprise wow surprised wtf surprisingly\n",
      "love : love mothers mommies moms loved\n",
      "anger : confuzzled fridaaaayyyyy aaaaaaaaaaa transtelecom filthy\n",
      "boredom : squeaking ouuut cleanin sooooooo candyland3\n",
      "worry : worried poor throat hurts sick\n"
     ]
    }
   ],
   "source": [
    "for k in by_cls:\n",
    "    words = [x[0] for x in by_cls[k].most_common(5)]\n",
    "    print(k, ':', ' '.join(words))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training a deep model\n",
    "\n",
    "Now that we've seen how well a simple linear model can do, let's see if we can do any better with a deep learning model.  In this case, we don't have an excessive amount of training data: this constrains the models we can train effectively: use too big of a model, and we'll end up overfitting our data.\n",
    "\n",
    "We'll start with a CNN."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40000, 167, 100)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from itertools import chain\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "import numpy as np\n",
    "\n",
    "chars = list(sorted(set(chain(*emotion_df['content']))))\n",
    "char_to_idx = {ch: idx for idx, ch in enumerate(chars)}\n",
    "max_sequence_len = max(len(x) for x in emotion_df['content'])\n",
    "\n",
    "char_vectors = []\n",
    "for txt in emotion_df['content']:\n",
    "    vec = np.zeros((max_sequence_len, len(char_to_idx)))\n",
    "    vec[np.arange(len(txt)), [char_to_idx[ch] for ch in txt]] = 1\n",
    "    char_vectors.append(vec)\n",
    "char_vectors = np.asarray(char_vectors)\n",
    "char_vectors = pad_sequences(char_vectors)\n",
    "labels = label_encoder.transform(emotion_df['sentiment'])\n",
    "\n",
    "\n",
    "def split(lst):\n",
    "    training_count = int(0.9 * len(char_vectors))\n",
    "    return lst[:training_count], lst[training_count:]\n",
    "\n",
    "training_char_vectors, test_char_vectors = split(char_vectors)\n",
    "training_labels, test_labels = split(labels)\n",
    "\n",
    "char_vectors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input (InputLayer)           (None, 167, 100)          0         \n",
      "_________________________________________________________________\n",
      "conv1d_62 (Conv1D)           (None, 162, 128)          76928     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_59 (MaxPooling (None, 27, 128)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_63 (Conv1D)           (None, 22, 256)           196864    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_60 (MaxPooling (None, 3, 256)            0         \n",
      "_________________________________________________________________\n",
      "flatten_18 (Flatten)         (None, 768)               0         \n",
      "_________________________________________________________________\n",
      "dense_42 (Dense)             (None, 128)               98432     \n",
      "_________________________________________________________________\n",
      "dense_43 (Dense)             (None, 13)                1677      \n",
      "=================================================================\n",
      "Total params: 373,901\n",
      "Trainable params: 373,901\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Input, Conv1D, MaxPooling1D, Flatten, Dense, Dropout, Merge, LSTM\n",
    "from keras.models import Model\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras import regularizers\n",
    "\n",
    "def create_char_cnn_model(num_chars, max_sequence_len, num_labels):\n",
    "    char_input = Input(shape=(max_sequence_len, num_chars), name='input')\n",
    "    \n",
    "    conv_1x = Conv1D(128, 6, activation='relu', padding='valid')(char_input)\n",
    "    max_pool_1x = MaxPooling1D(6)(conv_1x)\n",
    "    conv_2x = Conv1D(256, 6, activation='relu', padding='valid')(max_pool_1x)\n",
    "    max_pool_2x = MaxPooling1D(6)(conv_2x)\n",
    "\n",
    "    flatten = Flatten()(max_pool_2x)\n",
    "    dense = Dense(128, \n",
    "                  activation='relu',\n",
    "                  kernel_regularizer=regularizers.l2(0.01))(flatten)\n",
    "    preds = Dense(num_labels, activation='softmax')(dense)\n",
    "\n",
    "    model = Model(char_input, preds)\n",
    "    model.compile(loss='sparse_categorical_crossentropy',\n",
    "                  optimizer='rmsprop',\n",
    "                  metrics=['acc'])\n",
    "    return model\n",
    "\n",
    "char_cnn_model = create_char_cnn_model(len(char_to_idx), char_vectors.shape[1], len(label_encoder.classes_))\n",
    "char_cnn_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "36000/36000 [==============================] - 2s - loss: 2.1909 - acc: 0.2315     \n",
      "Epoch 2/20\n",
      "36000/36000 [==============================] - 1s - loss: 2.1408 - acc: 0.2471     - ETA: 0s - loss: 2.1401 - acc: 0.24\n",
      "Epoch 3/20\n",
      "36000/36000 [==============================] - 1s - loss: 2.1294 - acc: 0.2501     \n",
      "Epoch 4/20\n",
      "36000/36000 [==============================] - 1s - loss: 2.1141 - acc: 0.2589     \n",
      "Epoch 5/20\n",
      "36000/36000 [==============================] - 1s - loss: 2.0845 - acc: 0.2657     \n",
      "Epoch 6/20\n",
      "36000/36000 [==============================] - 1s - loss: 2.0609 - acc: 0.2801     \n",
      "Epoch 7/20\n",
      "36000/36000 [==============================] - 1s - loss: 2.0346 - acc: 0.2895     \n",
      "Epoch 8/20\n",
      "36000/36000 [==============================] - 1s - loss: 2.0091 - acc: 0.3023     \n",
      "Epoch 9/20\n",
      "36000/36000 [==============================] - 1s - loss: 1.9866 - acc: 0.3125     \n",
      "Epoch 10/20\n",
      "36000/36000 [==============================] - 1s - loss: 1.9652 - acc: 0.3217     \n",
      "Epoch 11/20\n",
      "36000/36000 [==============================] - 1s - loss: 1.9422 - acc: 0.3292     \n",
      "Epoch 12/20\n",
      "36000/36000 [==============================] - 1s - loss: 1.9189 - acc: 0.3399     \n",
      "Epoch 13/20\n",
      "36000/36000 [==============================] - 1s - loss: 1.9005 - acc: 0.3464     \n",
      "Epoch 14/20\n",
      "36000/36000 [==============================] - 1s - loss: 1.8786 - acc: 0.3545     - ETA: 0s - loss: 1.8768 -\n",
      "Epoch 15/20\n",
      "36000/36000 [==============================] - 1s - loss: 1.8551 - acc: 0.3616     \n",
      "Epoch 16/20\n",
      "36000/36000 [==============================] - 1s - loss: 1.8386 - acc: 0.3691     \n",
      "Epoch 17/20\n",
      "36000/36000 [==============================] - 1s - loss: 1.8188 - acc: 0.3729     \n",
      "Epoch 18/20\n",
      "36000/36000 [==============================] - 1s - loss: 1.7984 - acc: 0.3814     \n",
      "Epoch 19/20\n",
      "36000/36000 [==============================] - 1s - loss: 1.7704 - acc: 0.3889     \n",
      "Epoch 20/20\n",
      "36000/36000 [==============================] - 1s - loss: 1.7602 - acc: 0.3959     \n",
      "3680/4000 [==========================>...] - ETA: 0s"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[2.0750380573272706, 0.30449999999999999]"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_cnn_model.fit(training_char_vectors, training_labels, epochs=20, batch_size=1024)\n",
    "char_cnn_model.evaluate(test_char_vectors, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input (InputLayer)               (None, 167, 100)      0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_52 (Conv1D)               (None, 163, 128)      64128                                        \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_54 (Conv1D)               (None, 162, 128)      76928                                        \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_56 (Conv1D)               (None, 161, 128)      89728                                        \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_49 (MaxPooling1D)  (None, 32, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_51 (MaxPooling1D)  (None, 27, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_53 (MaxPooling1D)  (None, 23, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_40 (Dropout)             (None, 32, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_42 (Dropout)             (None, 27, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_44 (Dropout)             (None, 23, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_53 (Conv1D)               (None, 28, 128)       82048                                        \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_55 (Conv1D)               (None, 22, 128)       98432                                        \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_57 (Conv1D)               (None, 17, 128)       114816                                       \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_50 (MaxPooling1D)  (None, 5, 128)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_52 (MaxPooling1D)  (None, 3, 128)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_54 (MaxPooling1D)  (None, 2, 128)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_41 (Dropout)             (None, 5, 128)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_43 (Dropout)             (None, 3, 128)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_45 (Dropout)             (None, 2, 128)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_6 (Concatenate)      (None, 10, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_46 (Dropout)             (None, 10, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "flatten_15 (Flatten)             (None, 1280)          0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dense_36 (Dense)                 (None, 128)           163968                                       \n",
      "____________________________________________________________________________________________________\n",
      "dense_37 (Dense)                 (None, 13)            1677                                         \n",
      "====================================================================================================\n",
      "Total params: 691,725\n",
      "Trainable params: 691,725\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Input, Conv1D, MaxPooling1D, Flatten, Dense, Dropout, Merge, LSTM\n",
    "from keras.models import Model\n",
    "from keras.layers.merge import Concatenate\n",
    "\n",
    "def create_char_cnn_model(num_chars, max_sequence_len, num_labels):\n",
    "    char_input = Input(shape=(max_sequence_len, num_chars), name='input')\n",
    "    \n",
    "    layers = []\n",
    "    for window in (5, 6, 7):\n",
    "        conv_1x = Conv1D(128, window, activation='relu', padding='valid')(char_input)\n",
    "        max_pool_1x = MaxPooling1D(window)(conv_1x)\n",
    "        dropout_1x = Dropout(0.3)(max_pool_1x)\n",
    "        conv_2x = Conv1D(128, window, activation='relu', padding='valid')(dropout_1x)\n",
    "        max_pool_2x = MaxPooling1D(window)(conv_2x)\n",
    "        dropout_2x = Dropout(0.3)(max_pool_2x)\n",
    "        layers.append(dropout_2x)\n",
    "\n",
    "    if len(layers) > 1:\n",
    "        merged = Concatenate(axis=1)(layers)\n",
    "    else:\n",
    "        merged = layers[0]\n",
    "\n",
    "    dropout = Dropout(0.3)(merged)\n",
    "    \n",
    "    flatten = Flatten()(dropout)\n",
    "    dense = Dense(128, activation='relu')(flatten)\n",
    "    preds = Dense(num_labels, activation='softmax')(dense)\n",
    "\n",
    "    model = Model(char_input, preds)\n",
    "    model.compile(loss='sparse_categorical_crossentropy',\n",
    "                  optimizer='rmsprop',\n",
    "                  metrics=['acc'])\n",
    "    return model\n",
    "\n",
    "char_cnn_model = create_char_cnn_model(len(char_to_idx), char_vectors.shape[1], len(label_encoder.classes_))\n",
    "char_cnn_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "36000/36000 [==============================] - 4s - loss: 2.1944 - acc: 0.2274     \n",
      "Epoch 2/20\n",
      "36000/36000 [==============================] - 3s - loss: 2.1351 - acc: 0.2493     \n",
      "Epoch 3/20\n",
      "36000/36000 [==============================] - 3s - loss: 2.1146 - acc: 0.2594     \n",
      "Epoch 4/20\n",
      "36000/36000 [==============================] - 3s - loss: 2.0697 - acc: 0.2777     \n",
      "Epoch 5/20\n",
      "36000/36000 [==============================] - 3s - loss: 2.0311 - acc: 0.2994     \n",
      "Epoch 6/20\n",
      "36000/36000 [==============================] - 3s - loss: 2.0004 - acc: 0.3108     \n",
      "Epoch 7/20\n",
      "36000/36000 [==============================] - 3s - loss: 1.9701 - acc: 0.3242     \n",
      "Epoch 8/20\n",
      "36000/36000 [==============================] - 3s - loss: 1.9473 - acc: 0.3342     \n",
      "Epoch 9/20\n",
      "36000/36000 [==============================] - 3s - loss: 1.9248 - acc: 0.3398     \n",
      "Epoch 10/20\n",
      "36000/36000 [==============================] - 3s - loss: 1.9022 - acc: 0.3493     \n",
      "Epoch 11/20\n",
      "36000/36000 [==============================] - 3s - loss: 1.8889 - acc: 0.3514     \n",
      "Epoch 12/20\n",
      "36000/36000 [==============================] - 3s - loss: 1.8644 - acc: 0.3642     \n",
      "Epoch 13/20\n",
      "36000/36000 [==============================] - 3s - loss: 1.8440 - acc: 0.3699     \n",
      "Epoch 14/20\n",
      "36000/36000 [==============================] - 3s - loss: 1.8260 - acc: 0.3775     \n",
      "Epoch 15/20\n",
      "36000/36000 [==============================] - 3s - loss: 1.8064 - acc: 0.3840     \n",
      "Epoch 16/20\n",
      "36000/36000 [==============================] - 3s - loss: 1.7846 - acc: 0.3899     \n",
      "Epoch 17/20\n",
      "36000/36000 [==============================] - 3s - loss: 1.7601 - acc: 0.3996     \n",
      "Epoch 18/20\n",
      "36000/36000 [==============================] - 3s - loss: 1.7482 - acc: 0.4040     \n",
      "Epoch 19/20\n",
      "36000/36000 [==============================] - 3s - loss: 1.7294 - acc: 0.4088     \n",
      "Epoch 20/20\n",
      "36000/36000 [==============================] - 3s - loss: 1.7060 - acc: 0.4136     \n",
      "3744/4000 [===========================>..] - ETA: 0s"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.9779343624114991, 0.35549999999999998]"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_cnn_model.fit(training_char_vectors, training_labels, epochs=20, batch_size=1024)\n",
    "char_cnn_model.evaluate(test_char_vectors, test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Featurizing and preparing our data\n",
    "\n",
    "Just like we did when computing word embeddings, we want to featurize our data so we can classify it effectively."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.preprocessing.text import one_hot\n",
    "\n",
    "VOCAB_SIZE = 50000\n",
    "tokenizer = Tokenizer(num_words=VOCAB_SIZE)\n",
    "tokenizer.fit_on_texts(emotion_df['content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This may take a while to load\n",
    "\n",
    "w2v, idf = nb_utils.load_w2v(tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tokens = tokenizer.texts_to_sequences(emotion_df['content'])\n",
    "tokens = pad_sequences(tokens)\n",
    "\n",
    "\n",
    "\n",
    "training_tokens, training_labels = tokens[:training_count], labels[:training_count]\n",
    "test_tokens, test_labels = tokens[training_count:], labels[training_count:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "message (InputLayer)             (None, None)          0                                            \n",
      "____________________________________________________________________________________________________\n",
      "message_vec/embedding (Embedding (None, None, 300)     15000000                                     \n",
      "____________________________________________________________________________________________________\n",
      "masking_1 (Masking)              (None, None, 300)     0                                            \n",
      "____________________________________________________________________________________________________\n",
      "message_idf/embedding (Embedding (None, None, 1)       50000                                        \n",
      "____________________________________________________________________________________________________\n",
      "combine_and_sum (Lambda)         (None, 300)           0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dense_23 (Dense)                 (None, 128)           38528                                        \n",
      "____________________________________________________________________________________________________\n",
      "dense_24 (Dense)                 (None, 13)            1677                                         \n",
      "====================================================================================================\n",
      "Total params: 15,090,205\n",
      "Trainable params: 40,205\n",
      "Non-trainable params: 15,050,000\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras import layers, models\n",
    "import keras.backend as K\n",
    "\n",
    "\n",
    "def make_embedding(name, vocab_size, embedding_size, weights=None, mask_zero=True):\n",
    "    if weights is not None:\n",
    "        return layers.Embedding(mask_zero=mask_zero, input_dim=vocab_size, \n",
    "                                output_dim=weights.shape[1], \n",
    "                                weights=[weights], trainable=False, \n",
    "                                name='%s/embedding' % name)\n",
    "    else:\n",
    "        return layers.Embedding(mask_zero=mask_zero, input_dim=vocab_size, \n",
    "                                output_dim=embedding_size,\n",
    "                                name='%s/embedding' % name)\n",
    "\n",
    "def create_unigram_model(vocab_size, embedding_size=None, embedding_weights=None, idf_weights=None):\n",
    "    assert not (embedding_size is None and embedding_weights is None)\n",
    "    message = layers.Input(shape=(None,), dtype='int32', name='message')\n",
    "    \n",
    "    embedding = make_embedding('message_vec', vocab_size, embedding_size, embedding_weights)\n",
    "    idf = make_embedding('message_idf', vocab_size, embedding_size, idf_weights)\n",
    "\n",
    "    mask = layers.Masking(mask_value=0)\n",
    "    def _combine_and_sum(args):\n",
    "        embedding, idf = args\n",
    "        return K.sum(embedding * K.abs(idf), axis=1)\n",
    "\n",
    "    sum_layer = layers.Lambda(_combine_and_sum, name='combine_and_sum')\n",
    "    sum_msg = sum_layer([mask(embedding(message)), idf(message)])\n",
    "    fc1 = layers.Dense(units=128, activation='relu')(sum_msg)\n",
    "    categories = layers.Dense(units=len(label_encoder.classes_), activation='softmax')(fc1)\n",
    "    \n",
    "    model = models.Model(\n",
    "        inputs=[message],\n",
    "        outputs=categories,\n",
    "    )\n",
    "    \n",
    "    model.compile(loss='sparse_categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "unigram_model = create_unigram_model(vocab_size=VOCAB_SIZE,\n",
    "                                     embedding_weights=w2v,\n",
    "                                     idf_weights=idf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "36000/36000 [==============================] - 2s - loss: 2.2822 - acc: 0.2821     \n",
      "Epoch 2/10\n",
      "36000/36000 [==============================] - 1s - loss: 1.9963 - acc: 0.3287     \n",
      "Epoch 3/10\n",
      "36000/36000 [==============================] - 1s - loss: 1.9501 - acc: 0.3419     \n",
      "Epoch 4/10\n",
      "36000/36000 [==============================] - 1s - loss: 1.9169 - acc: 0.3497     \n",
      "Epoch 5/10\n",
      "36000/36000 [==============================] - 1s - loss: 1.8967 - acc: 0.3574     \n",
      "Epoch 6/10\n",
      "36000/36000 [==============================] - 1s - loss: 1.8781 - acc: 0.3630     \n",
      "Epoch 7/10\n",
      "36000/36000 [==============================] - 1s - loss: 1.8597 - acc: 0.3699     \n",
      "Epoch 8/10\n",
      "36000/36000 [==============================] - 1s - loss: 1.8392 - acc: 0.3733     \n",
      "Epoch 9/10\n",
      "36000/36000 [==============================] - 1s - loss: 1.8259 - acc: 0.3797     \n",
      "Epoch 10/10\n",
      "36000/36000 [==============================] - 1s - loss: 1.8095 - acc: 0.3834     \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f5ffe0344e0>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unigram_model.fit(training_tokens, training_labels, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2.3744175682067872, 0.29449999999999998]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unigram_model.evaluate(test_tokens, test_labels, verbose=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning Embeddings\n",
    "\n",
    "It looks like our model with pre-trained embeddings isn't doing much better than the linear models.\n",
    "\n",
    "We can also try training a model \"from scratch\", and learn the word embeddings from our training data.  Note that we use a small embedding size here to speed up training and to try to avoid overfitting.\n",
    "\n",
    "Only training for 10 epochs stops the model while it is still improving on the training set, but prevents it\n",
    "from overfitting.  We can formalize this by using a validation set and early stopping."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "message (InputLayer)             (None, None)          0                                            \n",
      "____________________________________________________________________________________________________\n",
      "message_vec/embedding (Embedding (None, None, 25)      1250000                                      \n",
      "____________________________________________________________________________________________________\n",
      "masking_2 (Masking)              (None, None, 25)      0                                            \n",
      "____________________________________________________________________________________________________\n",
      "message_idf/embedding (Embedding (None, None, 25)      1250000                                      \n",
      "____________________________________________________________________________________________________\n",
      "combine_and_sum (Lambda)         (None, 25)            0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dense_25 (Dense)                 (None, 128)           3328                                         \n",
      "____________________________________________________________________________________________________\n",
      "dense_26 (Dense)                 (None, 13)            1677                                         \n",
      "====================================================================================================\n",
      "Total params: 2,505,005\n",
      "Trainable params: 2,505,005\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "learned_embeddings_model = create_unigram_model(vocab_size=VOCAB_SIZE, embedding_size=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "36000/36000 [==============================] - 2s - loss: 2.1545 - acc: 0.2497     \n",
      "Epoch 2/10\n",
      "36000/36000 [==============================] - 0s - loss: 2.0021 - acc: 0.3127     \n",
      "Epoch 3/10\n",
      "36000/36000 [==============================] - 0s - loss: 1.9004 - acc: 0.3492     \n",
      "Epoch 4/10\n",
      "36000/36000 [==============================] - 0s - loss: 1.8343 - acc: 0.3781     \n",
      "Epoch 5/10\n",
      "36000/36000 [==============================] - 0s - loss: 1.7814 - acc: 0.3974     \n",
      "Epoch 6/10\n",
      "36000/36000 [==============================] - 0s - loss: 1.7334 - acc: 0.4144     \n",
      "Epoch 7/10\n",
      "36000/36000 [==============================] - 0s - loss: 1.6845 - acc: 0.4343     \n",
      "Epoch 8/10\n",
      "36000/36000 [==============================] - 0s - loss: 1.6362 - acc: 0.4504     \n",
      "Epoch 9/10\n",
      "36000/36000 [==============================] - 0s - loss: 1.5850 - acc: 0.4696     \n",
      "Epoch 10/10\n",
      "36000/36000 [==============================] - 0s - loss: 1.5316 - acc: 0.4883     \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f6002cf8e10>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learned_embeddings_model.fit(training_tokens, training_labels, epochs=10, batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2.002617612838745, 0.36399999999999999]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note the test set accuracy is lower than that on the training set.\n",
    "\n",
    "learned_embeddings_model.evaluate(test_tokens, test_labels, verbose=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# More Complex Models\n",
    "\n",
    "As with our previous task, we can try using more powerful models to classify our text.  In this case, the limited training data and text size limit their effectiveness."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_cnn_model(vocab_size, embedding_size=None, embedding_weights=None):\n",
    "    message = layers.Input(shape=(None,), dtype='int32', name='title')\n",
    "    \n",
    "    # The convolution layer in keras does not support masking, so we just allow\n",
    "    # the embedding layer to learn an explicit value.\n",
    "    embedding = make_embedding('message_vec', vocab_size, embedding_size, embedding_weights,\n",
    "                              mask_zero=False)\n",
    "\n",
    "    def _combine_sum(v):\n",
    "        return K.sum(v, axis=1)\n",
    "\n",
    "    cnn_1 = layers.Convolution1D(128, 3)\n",
    "    cnn_2 = layers.Convolution1D(128, 3)\n",
    "    cnn_3 = layers.Convolution1D(128, 3)\n",
    "    \n",
    "    global_pool = layers.GlobalMaxPooling1D()\n",
    "    local_pool = layers.MaxPooling1D(strides=1, pool_size=3)\n",
    "\n",
    "    cnn_encoding = global_pool(cnn_3(local_pool(cnn_2(local_pool(cnn_1(embedding(message)))))))\n",
    "    fc1 = layers.Dense(units=128, activation='elu')(cnn_encoding)\n",
    "    categories = layers.Dense(units=len(label_encoder.classes_), activation='softmax')(fc1)\n",
    "    model = models.Model(\n",
    "        inputs=[message],\n",
    "        outputs=[categories],\n",
    "    )\n",
    "    model.compile(loss='sparse_categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "title (InputLayer)           (None, None)              0         \n",
      "_________________________________________________________________\n",
      "message_vec/embedding (Embed (None, None, 300)         15000000  \n",
      "_________________________________________________________________\n",
      "conv1d_41 (Conv1D)           (None, None, 128)         115328    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_41 (MaxPooling (None, None, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv1d_42 (Conv1D)           (None, None, 128)         49280     \n",
      "_________________________________________________________________\n",
      "conv1d_43 (Conv1D)           (None, None, 128)         49280     \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_1 (Glob (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_27 (Dense)             (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dense_28 (Dense)             (None, 13)                1677      \n",
      "=================================================================\n",
      "Total params: 15,232,077\n",
      "Trainable params: 232,077\n",
      "Non-trainable params: 15,000,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "cnn_model = create_cnn_model(VOCAB_SIZE, embedding_weights=w2v)\n",
    "cnn_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "36000/36000 [==============================] - 3s - loss: 1.9843 - acc: 0.3157     \n",
      "Epoch 2/10\n",
      "36000/36000 [==============================] - 2s - loss: 1.8778 - acc: 0.3522     \n",
      "Epoch 3/10\n",
      "36000/36000 [==============================] - 2s - loss: 1.8190 - acc: 0.3731     \n",
      "Epoch 4/10\n",
      "36000/36000 [==============================] - 2s - loss: 1.7614 - acc: 0.3927     \n",
      "Epoch 5/10\n",
      "36000/36000 [==============================] - 2s - loss: 1.6857 - acc: 0.4214     \n",
      "Epoch 6/10\n",
      "36000/36000 [==============================] - 2s - loss: 1.5980 - acc: 0.4489     \n",
      "Epoch 7/10\n",
      "36000/36000 [==============================] - 2s - loss: 1.4942 - acc: 0.4806     \n",
      "Epoch 8/10\n",
      "36000/36000 [==============================] - 2s - loss: 1.3862 - acc: 0.5167     \n",
      "Epoch 9/10\n",
      "36000/36000 [==============================] - 2s - loss: 1.2714 - acc: 0.5567     \n",
      "Epoch 10/10\n",
      "36000/36000 [==============================] - 3s - loss: 1.1676 - acc: 0.5933     \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f6001ceb128>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn_model.fit(training_tokens, training_labels, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3232/4000 [=======================>......] - ETA: 0s"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[2.5513558645248411, 0.32450000000000001]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn_model.evaluate(test_tokens, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_lstm_model(vocab_size, embedding_size=None, embedding_weights=None):\n",
    "    message = layers.Input(shape=(None,), dtype='int32', name='title')\n",
    "    embedding = make_embedding('message_vec', vocab_size, embedding_size, embedding_weights)(message)\n",
    "\n",
    "    lstm_1 = layers.LSTM(units=128, return_sequences=False)(embedding)\n",
    "#     lstm_2 = layers.LSTM(units=128, return_sequences=False)(lstm_1)\n",
    "    category = layers.Dense(units=len(label_encoder.classes_), activation='softmax')(lstm_1)\n",
    "    \n",
    "    model = models.Model(\n",
    "        inputs=[message],\n",
    "        outputs=[category],\n",
    "    )\n",
    "    model.compile(loss='sparse_categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "title (InputLayer)           (None, None)              0         \n",
      "_________________________________________________________________\n",
      "message_vec/embedding (Embed (None, None, 300)         15000000  \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 128)               219648    \n",
      "_________________________________________________________________\n",
      "dense_29 (Dense)             (None, 13)                1677      \n",
      "=================================================================\n",
      "Total params: 15,221,325\n",
      "Trainable params: 221,325\n",
      "Non-trainable params: 15,000,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "lstm_model = create_lstm_model(VOCAB_SIZE, embedding_weights=w2v)\n",
    "lstm_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "36000/36000 [==============================] - 22s - loss: 2.0147 - acc: 0.3125    \n",
      "Epoch 2/10\n",
      "36000/36000 [==============================] - 19s - loss: 1.8909 - acc: 0.3542    \n",
      "Epoch 3/10\n",
      "36000/36000 [==============================] - 19s - loss: 1.8447 - acc: 0.3700    \n",
      "Epoch 4/10\n",
      "36000/36000 [==============================] - 19s - loss: 1.8151 - acc: 0.3787    \n",
      "Epoch 5/10\n",
      "36000/36000 [==============================] - 19s - loss: 1.7889 - acc: 0.3860    \n",
      "Epoch 6/10\n",
      "36000/36000 [==============================] - 19s - loss: 1.7636 - acc: 0.3927    \n",
      "Epoch 7/10\n",
      "36000/36000 [==============================] - 19s - loss: 1.7409 - acc: 0.3996    \n",
      "Epoch 8/10\n",
      "36000/36000 [==============================] - 19s - loss: 1.7165 - acc: 0.4071    \n",
      "Epoch 9/10\n",
      "36000/36000 [==============================] - 19s - loss: 1.6898 - acc: 0.4179    \n",
      "Epoch 10/10\n",
      "36000/36000 [==============================] - 19s - loss: 1.6584 - acc: 0.4284    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f6067053f60>"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_model.fit(training_tokens, training_labels, epochs=10, batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3936/4000 [============================>.] - ETA: 0s"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.918457947731018, 0.36149999999999999]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_model.evaluate(test_tokens, test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparing our models\n",
    "\n",
    "Let's compare the predictions from our models on a sample of our data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = {\n",
    "    'lstm': lstm_model.predict(test_tokens[:100]),\n",
    "    'char_cnn': char_cnn_model.predict(test_char_vectors[:100]),\n",
    "    'cnn': cnn_model.predict(test_tokens[:100]),\n",
    "    'unigram': unigram_model.predict(test_tokens[:100]),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>true</th>\n",
       "      <th>lstm</th>\n",
       "      <th>cnn</th>\n",
       "      <th>char_cnn</th>\n",
       "      <th>unigram</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HAPPY MOTHER'S DAY to all of the wonderful women out there.  Have a great and relaxful day.</td>\n",
       "      <td>happiness</td>\n",
       "      <td>love</td>\n",
       "      <td>love</td>\n",
       "      <td>love</td>\n",
       "      <td>love</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>browsing thru adopting agencies, i'm gonna get some exotic kids</td>\n",
       "      <td>enthusiasm</td>\n",
       "      <td>neutral</td>\n",
       "      <td>neutral</td>\n",
       "      <td>neutral</td>\n",
       "      <td>happiness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I am tired of my phone. Walkman works like a charm, but l need better video and wap really. Thanks for yesterday and for buy...</td>\n",
       "      <td>love</td>\n",
       "      <td>worry</td>\n",
       "      <td>happiness</td>\n",
       "      <td>sadness</td>\n",
       "      <td>happiness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Happy Mother's Day to all the Mommiessss</td>\n",
       "      <td>love</td>\n",
       "      <td>love</td>\n",
       "      <td>love</td>\n",
       "      <td>love</td>\n",
       "      <td>love</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>@mattgarner haha what's up Matt ?</td>\n",
       "      <td>happiness</td>\n",
       "      <td>neutral</td>\n",
       "      <td>happiness</td>\n",
       "      <td>neutral</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>What's up!!? @guillermop</td>\n",
       "      <td>neutral</td>\n",
       "      <td>neutral</td>\n",
       "      <td>neutral</td>\n",
       "      <td>neutral</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>@KandyBee we shuld do  a dance like that its seriously the best thing haha. see yu tomoro.</td>\n",
       "      <td>fun</td>\n",
       "      <td>fun</td>\n",
       "      <td>love</td>\n",
       "      <td>happiness</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>@TravelTweetie I will go to sleep now. Might be awakened early w/breakfast tray from my 'spark' &amp;amp; my 'joper' w/their Dad...</td>\n",
       "      <td>happiness</td>\n",
       "      <td>neutral</td>\n",
       "      <td>worry</td>\n",
       "      <td>worry</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>@nak1a &amp;quot;If there's a camel up a hill&amp;quot; and &amp;quot;I'll give you plankton&amp;quot; ....HILARIOUS!!</td>\n",
       "      <td>happiness</td>\n",
       "      <td>happiness</td>\n",
       "      <td>happiness</td>\n",
       "      <td>happiness</td>\n",
       "      <td>worry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>@Bern_morley LOL I love your kids</td>\n",
       "      <td>love</td>\n",
       "      <td>love</td>\n",
       "      <td>love</td>\n",
       "      <td>love</td>\n",
       "      <td>love</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                           content  \\\n",
       "0                                      HAPPY MOTHER'S DAY to all of the wonderful women out there.  Have a great and relaxful day.   \n",
       "1                                                                  browsing thru adopting agencies, i'm gonna get some exotic kids   \n",
       "2  I am tired of my phone. Walkman works like a charm, but l need better video and wap really. Thanks for yesterday and for buy...   \n",
       "3                                                                                         Happy Mother's Day to all the Mommiessss   \n",
       "4                                                                                                @mattgarner haha what's up Matt ?   \n",
       "5                                                                                                         What's up!!? @guillermop   \n",
       "6                                       @KandyBee we shuld do  a dance like that its seriously the best thing haha. see yu tomoro.   \n",
       "7  @TravelTweetie I will go to sleep now. Might be awakened early w/breakfast tray from my 'spark' &amp; my 'joper' w/their Dad...   \n",
       "8                           @nak1a &quot;If there's a camel up a hill&quot; and &quot;I'll give you plankton&quot; ....HILARIOUS!!   \n",
       "9                                                                                                @Bern_morley LOL I love your kids   \n",
       "\n",
       "         true       lstm        cnn   char_cnn    unigram  \n",
       "0   happiness       love       love       love       love  \n",
       "1  enthusiasm    neutral    neutral    neutral  happiness  \n",
       "2        love      worry  happiness    sadness  happiness  \n",
       "3        love       love       love       love       love  \n",
       "4   happiness    neutral  happiness    neutral    neutral  \n",
       "5     neutral    neutral    neutral    neutral    neutral  \n",
       "6         fun        fun       love  happiness    neutral  \n",
       "7   happiness    neutral      worry      worry    neutral  \n",
       "8   happiness  happiness  happiness  happiness      worry  \n",
       "9        love       love       love       love       love  "
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make a dataframe just for test data\n",
    "\n",
    "pd.options.display.max_colwidth = 128\n",
    "test_df = emotion_df[training_count:training_count+100].reset_index()\n",
    "eval_df = pd.DataFrame({\n",
    "    'content': test_df['content'],\n",
    "    'true': test_df['sentiment'],\n",
    "    'lstm': [label_encoder.classes_[np.argmax(x)] for x in predictions['lstm']],\n",
    "    'cnn': [label_encoder.classes_[np.argmax(x)] for x in predictions['cnn']],\n",
    "    'char_cnn': [label_encoder.classes_[np.argmax(x)] for x in predictions['char_cnn']],    \n",
    "    'unigram': [label_encoder.classes_[np.argmax(x)] for x in predictions['unigram']],\n",
    "})\n",
    "eval_df = eval_df[['content', 'true', 'lstm', 'cnn', 'char_cnn', 'unigram']]\n",
    "eval_df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Qualitative Evaluation\n",
    "\n",
    "We can examine some of our error cases by hand.  Often, the models tend to agree when they make mistakes, and that the mistakes aren't unreasonable: this task would be challenging even for a human."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>true</th>\n",
       "      <th>lstm</th>\n",
       "      <th>cnn</th>\n",
       "      <th>char_cnn</th>\n",
       "      <th>unigram</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HAPPY MOTHER'S DAY to all of the wonderful women out there.  Have a great and relaxful day.</td>\n",
       "      <td>happiness</td>\n",
       "      <td>love</td>\n",
       "      <td>love</td>\n",
       "      <td>love</td>\n",
       "      <td>love</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>browsing thru adopting agencies, i'm gonna get some exotic kids</td>\n",
       "      <td>enthusiasm</td>\n",
       "      <td>neutral</td>\n",
       "      <td>neutral</td>\n",
       "      <td>neutral</td>\n",
       "      <td>happiness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I am tired of my phone. Walkman works like a charm, but l need better video and wap really. Thanks for yesterday and for buy...</td>\n",
       "      <td>love</td>\n",
       "      <td>worry</td>\n",
       "      <td>happiness</td>\n",
       "      <td>sadness</td>\n",
       "      <td>happiness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>@mattgarner haha what's up Matt ?</td>\n",
       "      <td>happiness</td>\n",
       "      <td>neutral</td>\n",
       "      <td>happiness</td>\n",
       "      <td>neutral</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>@TravelTweetie I will go to sleep now. Might be awakened early w/breakfast tray from my 'spark' &amp;amp; my 'joper' w/their Dad...</td>\n",
       "      <td>happiness</td>\n",
       "      <td>neutral</td>\n",
       "      <td>worry</td>\n",
       "      <td>worry</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>@davecandoit dude that honest to god happens to me all the time.. minus the trail mix.</td>\n",
       "      <td>sadness</td>\n",
       "      <td>neutral</td>\n",
       "      <td>happiness</td>\n",
       "      <td>neutral</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Happy Mother's Day to the tweetin' mamas  Nite tweeple!</td>\n",
       "      <td>worry</td>\n",
       "      <td>love</td>\n",
       "      <td>love</td>\n",
       "      <td>love</td>\n",
       "      <td>happiness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>On my way home...then SLEEP! Seeing Amber Pacific tomorow with the besties</td>\n",
       "      <td>happiness</td>\n",
       "      <td>neutral</td>\n",
       "      <td>neutral</td>\n",
       "      <td>happiness</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>@xoMusicLoverxo I'm using it in a story. I actually already wrote it but have to write the chapters before it.</td>\n",
       "      <td>relief</td>\n",
       "      <td>worry</td>\n",
       "      <td>neutral</td>\n",
       "      <td>worry</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>@poptrashmusic How's your dog?</td>\n",
       "      <td>surprise</td>\n",
       "      <td>neutral</td>\n",
       "      <td>neutral</td>\n",
       "      <td>neutral</td>\n",
       "      <td>happiness</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                            content  \\\n",
       "0                                       HAPPY MOTHER'S DAY to all of the wonderful women out there.  Have a great and relaxful day.   \n",
       "1                                                                   browsing thru adopting agencies, i'm gonna get some exotic kids   \n",
       "2   I am tired of my phone. Walkman works like a charm, but l need better video and wap really. Thanks for yesterday and for buy...   \n",
       "4                                                                                                 @mattgarner haha what's up Matt ?   \n",
       "7   @TravelTweetie I will go to sleep now. Might be awakened early w/breakfast tray from my 'spark' &amp; my 'joper' w/their Dad...   \n",
       "10                                           @davecandoit dude that honest to god happens to me all the time.. minus the trail mix.   \n",
       "12                                                                          Happy Mother's Day to the tweetin' mamas  Nite tweeple!   \n",
       "13                                                       On my way home...then SLEEP! Seeing Amber Pacific tomorow with the besties   \n",
       "14                   @xoMusicLoverxo I'm using it in a story. I actually already wrote it but have to write the chapters before it.   \n",
       "15                                                                                                   @poptrashmusic How's your dog?   \n",
       "\n",
       "          true     lstm        cnn   char_cnn    unigram  \n",
       "0    happiness     love       love       love       love  \n",
       "1   enthusiasm  neutral    neutral    neutral  happiness  \n",
       "2         love    worry  happiness    sadness  happiness  \n",
       "4    happiness  neutral  happiness    neutral    neutral  \n",
       "7    happiness  neutral      worry      worry    neutral  \n",
       "10     sadness  neutral  happiness    neutral    neutral  \n",
       "12       worry     love       love       love  happiness  \n",
       "13   happiness  neutral    neutral  happiness    neutral  \n",
       "14      relief    worry    neutral      worry    neutral  \n",
       "15    surprise  neutral    neutral    neutral  happiness  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_df[eval_df['lstm'] != eval_df['true']].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "emotion_df.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyzing Tweets\n",
    "\n",
    "We can gather a sample of Twitter data using the Twitter API (https://dev.twitter.com).  To do so, we'll need to create a Twitter application and get credentials for it.  You can do this manually at https://app.twitter.com.  Once you have an app, go to the \"Key and Access Tokens\" tab to find your credentials."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import twitter\n",
    "import emoji"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Fill these in!\n",
    "\n",
    "CONSUMER_KEY = 'xbMuxcJpRTiVGt2C2EYnA'\n",
    "CONSUMER_SECRET = '2DbQTsvIptkPTdaUcos8DDvQH9fzO0hNjJpUT2uVzQ'\n",
    "ACCESS_TOKEN = '7319442-EDm4CPxL7W4KkZcGWRMJNVHp88W5OH9vgblu898fg'\n",
    "ACCESS_SECRET = '5ZxJSbqXhG7uhgXzTFWf9XhkfsxxinlPRXyDTzbA9w'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "api = twitter.Twitter(\n",
    "    auth=twitter.OAuth(\n",
    "        consumer_key=CONSUMER_KEY,\n",
    "        consumer_secret=CONSUMER_SECRET,\n",
    "        token=ACCESS_TOKEN,\n",
    "        token_secret=ACCESS_SECRET,\n",
    "    ))\n",
    "\n",
    "stream = twitter.TwitterStream(\n",
    "    auth=twitter.OAuth(\n",
    "        consumer_key=CONSUMER_KEY,\n",
    "        consumer_secret=CONSUMER_SECRET,\n",
    "        token=ACCESS_TOKEN,\n",
    "        token_secret=ACCESS_SECRET,\n",
    "    ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import itertools\n",
    "def has_emoji(tweet):\n",
    "    if tweet.get('lang') != 'en':\n",
    "        return False\n",
    "    return any(ch for ch in tweet.get('text', '') if ch in emoji.UNICODE_EMOJI)\n",
    "\n",
    "%time st = list(itertools.islice(filter(has_emoji, stream.statuses.sample()), 0, 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "len(st), [t.get('text', None) for t in st][:10]"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}