07.2 Emoji Suggestions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyzing Tweets\n",
    "\n",
    "We can gather a sample of Twitter data using the Twitter API (https://dev.twitter.com).  To do so, we'll need to create a Twitter application and get credentials for it.  You can do this manually at https://app.twitter.com.  Once you have an app, go to the \"Key and Access Tokens\" tab to find your credentials."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import twitter\n",
    "import emoji\n",
    "import itertools\n",
    "import pandas as pd\n",
    "from itertools import chain\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.preprocessing.text import one_hot\n",
    "import keras.callbacks\n",
    "import json\n",
    "\n",
    "import os\n",
    "import nb_utils\n",
    "from keras.layers import Input, Conv1D, MaxPooling1D, Flatten, Dense, Dropout, Merge, LSTM, Embedding, GlobalMaxPooling1D\n",
    "from keras.models import Model\n",
    "from keras.layers.merge import Concatenate, Average\n",
    "\n",
    "from gensim.models import Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Fill these in!\n",
    "\n",
    "CONSUMER_KEY = 'xbMuxcJpRTiVGt2C2EYnA'\n",
    "CONSUMER_SECRET = '2DbQTsvIptkPTdaUcos8DDvQH9fzO0hNjJpUT2uVzQ'\n",
    "ACCESS_TOKEN = '7319442-EDm4CPxL7W4KkZcGWRMJNVHp88W5OH9vgblu898fg'\n",
    "ACCESS_SECRET = '5ZxJSbqXhG7uhgXzTFWf9XhkfsxxinlPRXyDTzbA9w'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['RT @saru__0925: ほんとに人生１回でいいから踊ってみたい😩😩 https://t.co/dsM1jFgPpS']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auth=twitter.OAuth(\n",
    "    consumer_key=CONSUMER_KEY,\n",
    "    consumer_secret=CONSUMER_SECRET,\n",
    "    token=ACCESS_TOKEN,\n",
    "    token_secret=ACCESS_SECRET,\n",
    ")\n",
    "\n",
    "status_stream = twitter.TwitterStream(auth=auth).statuses\n",
    "\n",
    "[x['text'] for x in itertools.islice(status_stream.sample(), 0, 5) if x.get('text')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.78 s, sys: 608 ms, total: 5.39 s\n",
      "Wall time: 43.7 s\n"
     ]
    }
   ],
   "source": [
    "status_stream = twitter.TwitterStream(auth=auth).statuses\n",
    "\n",
    "def english_has_emoji(tweet):\n",
    "    if tweet.get('lang') != 'en':\n",
    "        return False\n",
    "    return any(ch for ch in tweet.get('text', '') if ch in emoji.UNICODE_EMOJI)\n",
    "\n",
    "%time tweets = list(itertools.islice(filter(english_has_emoji, status_stream.sample()), 0, 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stripped = []\n",
    "for tweet in tweets:\n",
    "    text = tweet['text']\n",
    "    emojis = {ch for ch in text if ch in emoji.UNICODE_EMOJI}\n",
    "    if len(emojis) == 1:\n",
    "        emoiji = emojis.pop()\n",
    "        text = ''.join(ch for ch in text if ch != emoiji)\n",
    "        stripped.append((text, emoiji))\n",
    "len(stripped)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using the CNN\n",
    "\n",
    "Let's see what the CNN of the previous chapter does on the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "😂    124823\n",
       "❤     43218\n",
       "😍     40566\n",
       "😭     35714\n",
       "😊     20076\n",
       "🙄     17963\n",
       "😩     16232\n",
       "🔥     15453\n",
       "🤔     15419\n",
       "💕     12026\n",
       "💯     11783\n",
       "😘     11065\n",
       "💀      9928\n",
       "✨      9886\n",
       "🙃      9405\n",
       "👀      7842\n",
       "😒      7019\n",
       "☺      6871\n",
       "😢      6846\n",
       "😳      6716\n",
       "💙      6616\n",
       "😎      6349\n",
       "😉      6272\n",
       "😅      6133\n",
       "😁      6010\n",
       "😌      5759\n",
       "😏      5623\n",
       "💖      5331\n",
       "😔      5244\n",
       "😴      4999\n",
       "      ...  \n",
       "🤤         1\n",
       "🦍         1\n",
       "🕡         1\n",
       "↩         1\n",
       "🏣         1\n",
       "🔀         1\n",
       "🤠         1\n",
       "📭         1\n",
       "👘         1\n",
       "📇         1\n",
       "📯         1\n",
       "📐         1\n",
       "🕣         1\n",
       "📂         1\n",
       "🕍         1\n",
       "🎽         1\n",
       "🈹         1\n",
       "🔣         1\n",
       "🏦         1\n",
       "🏗         1\n",
       "🥇         1\n",
       "🚡         1\n",
       "🛐         1\n",
       "🏺         1\n",
       "🎚         1\n",
       "🤷         1\n",
       "🔏         1\n",
       "🎑         1\n",
       "🤡         1\n",
       "🏬         1\n",
       "Name: emoji, Length: 989, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_tweets = pd.read_csv('data/emojis.csv')\n",
    "all_tweets['emoji'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "😂    124823\n",
       "❤     43218\n",
       "😍     40566\n",
       "😭     35714\n",
       "😊     20076\n",
       "🙄     17963\n",
       "😩     16232\n",
       "🔥     15453\n",
       "🤔     15419\n",
       "💕     12026\n",
       "💯     11783\n",
       "😘     11065\n",
       "💀      9928\n",
       "✨      9886\n",
       "🙃      9405\n",
       "👀      7842\n",
       "😒      7019\n",
       "☺      6871\n",
       "😢      6846\n",
       "😳      6716\n",
       "💙      6616\n",
       "😎      6349\n",
       "😉      6272\n",
       "😅      6133\n",
       "😁      6010\n",
       "😌      5759\n",
       "😏      5623\n",
       "💖      5331\n",
       "😔      5244\n",
       "😴      4999\n",
       "      ...  \n",
       "✌      1523\n",
       "📸      1496\n",
       "🎤      1487\n",
       "🌚      1452\n",
       "👅      1431\n",
       "🏈      1373\n",
       "🌟      1355\n",
       "⏩      1332\n",
       "❗      1325\n",
       "🔴      1304\n",
       "☕      1296\n",
       "👊      1273\n",
       "👇      1259\n",
       "❣      1254\n",
       "🎧      1246\n",
       "🎈      1210\n",
       "⏭      1198\n",
       "💫      1181\n",
       "↪      1157\n",
       "🤑      1152\n",
       "⚽      1141\n",
       "😹      1112\n",
       "😶      1108\n",
       "💦      1075\n",
       "😣      1074\n",
       "😥      1072\n",
       "🙁      1066\n",
       "🤕      1065\n",
       "😰      1013\n",
       "☀      1013\n",
       "Name: emoji, Length: 121, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets = all_tweets.groupby('emoji').filter(lambda c:len(c) > 1000)\n",
    "tweets['emoji'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Don't worry, my love, I don't get you wrong Don't get me wrong. We're connected with our hearts, souls, minds, bodies. If any doubt, we ask\""
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(tweets['text'], key=lambda t:len(t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "chars = list(sorted(set(chain(*tweets['text']))))\n",
    "char_to_idx = {ch: idx for idx, ch in enumerate(chars)}\n",
    "max_sequence_len = max(len(x) for x in tweets['text'])\n",
    "\n",
    "emojis = list(sorted(set(tweets['emoji'])))\n",
    "emoji_to_idx = {em: idx for idx, em in enumerate(emojis)}\n",
    "emojis[:10]\n",
    "\n",
    "train_tweets, test_tweets = train_test_split(tweets, test_size=0.1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         ..., \n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.]],\n",
       " \n",
       "        [[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         ..., \n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.]],\n",
       " \n",
       "        [[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 1.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         ..., \n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.]],\n",
       " \n",
       "        ..., \n",
       "        [[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 1.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         ..., \n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.]],\n",
       " \n",
       "        [[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         ..., \n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.]],\n",
       " \n",
       "        [[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 1.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         ..., \n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "         [ 0.,  0.,  0., ...,  0.,  0.,  0.]]]),\n",
       " array([  13.,   20.,  100.,   11.,   96.,   73.,   65.,   41.,   34.,   96.]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def data_generator(tweets, batch_size):\n",
    "    while True:\n",
    "        if batch_size is None:\n",
    "            batch = tweets\n",
    "            batch_size = batch.shape[0]\n",
    "        else:\n",
    "            batch = tweets.sample(batch_size)\n",
    "        X = np.zeros((batch_size, max_sequence_len, len(chars)))\n",
    "        y = np.zeros((batch_size,))\n",
    "        for row_idx, (_, row) in enumerate(batch.iterrows()):\n",
    "            y[row_idx] = emoji_to_idx[row['emoji']]\n",
    "            for ch_idx, ch in enumerate(row['text']):\n",
    "                X[row_idx, ch_idx, char_to_idx[ch]] = 1\n",
    "        yield X, y\n",
    "\n",
    "next(data_generator(tweets, 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "char_cnn_input (InputLayer)  (None, 139, 96)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_33 (Conv1D)           (None, 134, 128)          73856     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_27 (MaxPooling (None, 33, 128)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_34 (Conv1D)           (None, 28, 256)           196864    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_28 (MaxPooling (None, 7, 256)            0         \n",
      "_________________________________________________________________\n",
      "flatten_4 (Flatten)          (None, 1792)              0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 128)               229504    \n",
      "_________________________________________________________________\n",
      "char_cnn_predictions (Dense) (None, 121)               15609     \n",
      "=================================================================\n",
      "Total params: 515,833\n",
      "Trainable params: 515,833\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "def create_char_cnn_model(num_chars, max_sequence_len, num_labels):\n",
    "    char_input = Input(shape=(max_sequence_len, num_chars), name='char_cnn_input')\n",
    "    \n",
    "    conv_1x = Conv1D(128, 6, activation='relu', padding='valid')(char_input)\n",
    "    max_pool_1x = MaxPooling1D(4)(conv_1x)\n",
    "    conv_2x = Conv1D(256, 6, activation='relu', padding='valid')(max_pool_1x)\n",
    "    max_pool_2x = MaxPooling1D(4)(conv_2x)\n",
    "\n",
    "    flatten = Flatten()(max_pool_2x)\n",
    "    dense = Dense(128, activation='relu')(flatten)\n",
    "    preds = Dense(num_labels, activation='softmax', name='char_cnn_predictions')(dense)\n",
    "\n",
    "    model = Model(char_input, preds)\n",
    "    model.compile(loss='sparse_categorical_crossentropy',\n",
    "                  optimizer='rmsprop',\n",
    "                  metrics=['acc'])\n",
    "    return model\n",
    "\n",
    "char_cnn_model = create_char_cnn_model(len(char_to_idx), max_sequence_len, len(emojis))\n",
    "char_cnn_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "73s - loss: 3.6955 - acc: 0.2050\n",
      "Epoch 2/20\n",
      "65s - loss: 3.3265 - acc: 0.2616\n",
      "Epoch 3/20\n",
      "65s - loss: 3.1551 - acc: 0.2957\n",
      "Epoch 4/20\n",
      "65s - loss: 3.0320 - acc: 0.3192\n",
      "Epoch 5/20\n",
      "65s - loss: 2.9288 - acc: 0.3405\n",
      "Epoch 6/20\n",
      "65s - loss: 2.8463 - acc: 0.3574\n",
      "Epoch 7/20\n",
      "65s - loss: 2.7718 - acc: 0.3721\n",
      "Epoch 8/20\n",
      "65s - loss: 2.7119 - acc: 0.3843\n",
      "Epoch 9/20\n",
      "65s - loss: 2.6648 - acc: 0.3941\n",
      "Epoch 10/20\n",
      "65s - loss: 2.6333 - acc: 0.3999\n",
      "Epoch 11/20\n",
      "65s - loss: 2.5973 - acc: 0.4064\n",
      "Epoch 12/20\n",
      "65s - loss: 2.5609 - acc: 0.4137\n",
      "Epoch 13/20\n",
      "65s - loss: 2.5355 - acc: 0.4183\n",
      "Epoch 14/20\n",
      "64s - loss: 2.5126 - acc: 0.4218\n",
      "Epoch 15/20\n",
      "65s - loss: 2.4913 - acc: 0.4261\n",
      "Epoch 16/20\n",
      "65s - loss: 2.4705 - acc: 0.4303\n",
      "Epoch 17/20\n",
      "65s - loss: 2.4618 - acc: 0.4305\n",
      "Epoch 18/20\n",
      "65s - loss: 2.4440 - acc: 0.4337\n",
      "Epoch 19/20\n",
      "65s - loss: 2.4231 - acc: 0.4375\n",
      "Epoch 20/20\n",
      "65s - loss: 2.4104 - acc: 0.4395\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fd44a3e7c88>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "early = keras.callbacks.EarlyStopping(monitor='loss',\n",
    "                              min_delta=0.03,\n",
    "                              patience=2,\n",
    "                              verbose=0, mode='auto')\n",
    "\n",
    "BATCH_SIZE = 512\n",
    "char_cnn_model.fit_generator(\n",
    "    data_generator(train_tweets, batch_size=BATCH_SIZE),\n",
    "    epochs=20,\n",
    "    steps_per_epoch=len(train_tweets) / BATCH_SIZE,\n",
    "    verbose=2,\n",
    "    callbacks=[early]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[3.1370692166729248, 0.3634086277173913]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_cnn_model.evaluate_generator(\n",
    "    data_generator(test_tweets, batch_size=BATCH_SIZE),\n",
    "    steps=len(test_tweets) / BATCH_SIZE\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('zoo/07/emoji_chars.json', 'w') as fout:\n",
    "    json.dump({\n",
    "        'emojis': ''.join(emojis),\n",
    "        'char_to_idx': char_to_idx,\n",
    "        'max_sequence_len': max_sequence_len,\n",
    "    }, fout)\n",
    "char_cnn_model.save('zoo/07/char_cnn_model.h5')\n",
    "char_cnn_model.save_weights('zoo/07/char_cnn_model_weights.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>true</th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>96993</th>\n",
       "      <td>@lovereina_: I really wanna be caked up rn</td>\n",
       "      <td>🙄</td>\n",
       "      <td>😩</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21238</th>\n",
       "      <td>@PeopIe: Awesome friendship</td>\n",
       "      <td>😀</td>\n",
       "      <td>😀</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>637878</th>\n",
       "      <td>colourbee: Chibi Star and Marco, my first Star vs the forces of evil fanart yay :3</td>\n",
       "      <td>📷</td>\n",
       "      <td>😂</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>420818</th>\n",
       "      <td>@howluans: I love you so much, you make me happy just to smile. I hope one day you notice me here @camilacabello97</td>\n",
       "      <td>❤</td>\n",
       "      <td>❤</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223513</th>\n",
       "      <td>Go to hell</td>\n",
       "      <td>😈</td>\n",
       "      <td>😂</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>759685</th>\n",
       "      <td>My header poppin</td>\n",
       "      <td>😍</td>\n",
       "      <td>😂</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282387</th>\n",
       "      <td>Cesar is a fucking mongol</td>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>534106</th>\n",
       "      <td>@ComplexSports: OMFG</td>\n",
       "      <td>💀</td>\n",
       "      <td>💀</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249189</th>\n",
       "      <td>Let's have a chat now O &lt;== Here i am</td>\n",
       "      <td>⏭</td>\n",
       "      <td>👉</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>534534</th>\n",
       "      <td>There's a house centipede in my closet and since Ryan isn't here I have to kill it....</td>\n",
       "      <td>🙃</td>\n",
       "      <td>🙃</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                      text  \\\n",
       "96993                                                                           @lovereina_: I really wanna be caked up rn   \n",
       "21238                                                                                          @PeopIe: Awesome friendship   \n",
       "637878                                  colourbee: Chibi Star and Marco, my first Star vs the forces of evil fanart yay :3   \n",
       "420818  @howluans: I love you so much, you make me happy just to smile. I hope one day you notice me here @camilacabello97   \n",
       "223513                                                                                                          Go to hell   \n",
       "759685                                                                                                    My header poppin   \n",
       "282387                                                                                           Cesar is a fucking mongol   \n",
       "534106                                                                                                @ComplexSports: OMFG   \n",
       "249189                                                                               Let's have a chat now O <== Here i am   \n",
       "534534                              There's a house centipede in my closet and since Ryan isn't here I have to kill it....   \n",
       "\n",
       "       true pred  \n",
       "96993     🙄    😩  \n",
       "21238     😀    😀  \n",
       "637878    📷    😂  \n",
       "420818    ❤    ❤  \n",
       "223513    😈    😂  \n",
       "759685    😍    😂  \n",
       "282387    😂    😂  \n",
       "534106    💀    💀  \n",
       "249189    ⏭    👉  \n",
       "534534    🙃    🙃  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.options.display.max_colwidth = 128\n",
    "inspect_tweets = test_tweets.sample(100)\n",
    "predicted = char_cnn_model.predict_generator(data_generator(inspect_tweets, batch_size=None), steps=1)\n",
    "show = pd.DataFrame({\n",
    "    'text': inspect_tweets['text'],\n",
    "    'true': inspect_tweets['emoji'],\n",
    "    'pred': [emojis[np.argmax(x)] for x in predicted],\n",
    "})\n",
    "show = show[['text', 'true', 'pred']]\n",
    "show.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "char_cnn_input (InputLayer)      (None, 139, 96)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_35 (Conv1D)               (None, 136, 128)      49280                                        \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_37 (Conv1D)               (None, 135, 128)      61568                                        \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_39 (Conv1D)               (None, 134, 128)      73856                                        \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_29 (MaxPooling1D)  (None, 34, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_31 (MaxPooling1D)  (None, 33, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_33 (MaxPooling1D)  (None, 33, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_15 (Dropout)             (None, 34, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_17 (Dropout)             (None, 33, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_19 (Dropout)             (None, 33, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_36 (Conv1D)               (None, 31, 256)       131328                                       \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_38 (Conv1D)               (None, 29, 256)       164096                                       \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_40 (Conv1D)               (None, 28, 256)       196864                                       \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_30 (MaxPooling1D)  (None, 7, 256)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_32 (MaxPooling1D)  (None, 7, 256)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_34 (MaxPooling1D)  (None, 7, 256)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_16 (Dropout)             (None, 7, 256)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_18 (Dropout)             (None, 7, 256)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_20 (Dropout)             (None, 7, 256)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_5 (Concatenate)      (None, 21, 256)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_21 (Dropout)             (None, 21, 256)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "flatten_5 (Flatten)              (None, 5376)          0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dense_8 (Dense)                  (None, 128)           688256                                       \n",
      "____________________________________________________________________________________________________\n",
      "char_cnn_predictions (Dense)     (None, 121)           15609                                        \n",
      "====================================================================================================\n",
      "Total params: 1,380,857\n",
      "Trainable params: 1,380,857\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Input, Conv1D, MaxPooling1D, Flatten, Dense, Dropout, Merge, LSTM\n",
    "from keras.models import Model\n",
    "from keras.layers.merge import Concatenate\n",
    "\n",
    "def create_char_cnn_model2(num_chars, max_sequence_len, num_labels, drop_out=0.25):\n",
    "    char_input = Input(shape=(max_sequence_len, num_chars), name='char_cnn_input')\n",
    "    \n",
    "    layers = []\n",
    "    for window in (4, 5, 6):\n",
    "        conv_1x = Conv1D(128, window, activation='relu', padding='valid')(char_input)\n",
    "        max_pool_1x = MaxPooling1D(4)(conv_1x)\n",
    "        dropout_1x = Dropout(drop_out)(max_pool_1x)\n",
    "        conv_2x = Conv1D(256, window, activation='relu', padding='valid')(dropout_1x)\n",
    "        max_pool_2x = MaxPooling1D(4)(conv_2x)\n",
    "        dropout_2x = Dropout(drop_out)(max_pool_2x)\n",
    "        layers.append(dropout_2x)\n",
    "\n",
    "    merged = Concatenate(axis=1)(layers)\n",
    "\n",
    "    dropout = Dropout(drop_out)(merged)\n",
    "    \n",
    "    flatten = Flatten()(dropout)\n",
    "    dense = Dense(128, activation='relu')(flatten)\n",
    "    preds = Dense(num_labels, activation='softmax', name='char_cnn_predictions')(dense)\n",
    "\n",
    "    model = Model(char_input, preds)\n",
    "    model.compile(loss='sparse_categorical_crossentropy',\n",
    "                  optimizer='rmsprop',\n",
    "                  metrics=['acc'])\n",
    "    return model\n",
    "\n",
    "char_cnn_model2 = create_char_cnn_model2(len(char_to_idx), max_sequence_len, len(emojis))\n",
    "char_cnn_model2.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "59s - loss: 3.8784 - acc: 0.1843\n",
      "Epoch 2/30\n",
      "58s - loss: 3.5497 - acc: 0.2219\n",
      "Epoch 3/30\n",
      "58s - loss: 3.3917 - acc: 0.2501\n",
      "Epoch 4/30\n",
      "58s - loss: 3.2934 - acc: 0.2690\n",
      "Epoch 5/30\n",
      "58s - loss: 3.2190 - acc: 0.2835\n",
      "Epoch 6/30\n",
      "58s - loss: 3.1664 - acc: 0.2923\n",
      "Epoch 7/30\n",
      "58s - loss: 3.1136 - acc: 0.3020\n",
      "Epoch 8/30\n",
      "58s - loss: 3.0729 - acc: 0.3096\n",
      "Epoch 9/30\n",
      "58s - loss: 3.0430 - acc: 0.3154\n",
      "Epoch 10/30\n",
      "58s - loss: 3.0064 - acc: 0.3220\n",
      "Epoch 11/30\n",
      "58s - loss: 2.9784 - acc: 0.3268\n",
      "Epoch 12/30\n",
      "58s - loss: 2.9557 - acc: 0.3317\n",
      "Epoch 13/30\n",
      "58s - loss: 2.9331 - acc: 0.3365\n",
      "Epoch 14/30\n",
      "58s - loss: 2.9123 - acc: 0.3401\n",
      "Epoch 15/30\n",
      "58s - loss: 2.8850 - acc: 0.3453\n",
      "Epoch 16/30\n",
      "58s - loss: 2.8741 - acc: 0.3477\n",
      "Epoch 17/30\n",
      "58s - loss: 2.8603 - acc: 0.3491\n",
      "Epoch 18/30\n",
      "58s - loss: 2.8436 - acc: 0.3536\n",
      "Epoch 19/30\n",
      "58s - loss: 2.8279 - acc: 0.3562\n",
      "Epoch 20/30\n",
      "58s - loss: 2.8100 - acc: 0.3599\n",
      "Epoch 21/30\n",
      "58s - loss: 2.8045 - acc: 0.3600\n",
      "Epoch 22/30\n",
      "58s - loss: 2.7895 - acc: 0.3643\n",
      "Epoch 23/30\n",
      "58s - loss: 2.7777 - acc: 0.3652\n",
      "Epoch 24/30\n",
      "58s - loss: 2.7608 - acc: 0.3692\n",
      "Epoch 25/30\n",
      "58s - loss: 2.7560 - acc: 0.3693\n",
      "Epoch 26/30\n",
      "58s - loss: 2.7449 - acc: 0.3716\n",
      "Epoch 27/30\n",
      "58s - loss: 2.7389 - acc: 0.3725\n",
      "Epoch 28/30\n",
      "58s - loss: 2.7311 - acc: 0.3737\n",
      "Epoch 29/30\n",
      "58s - loss: 2.7232 - acc: 0.3754\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fd4437759e8>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BATCH_SIZE = 2048\n",
    "char_cnn_model2.fit_generator(\n",
    "    data_generator(train_tweets, batch_size=BATCH_SIZE),\n",
    "    epochs=30,\n",
    "    steps_per_epoch=len(train_tweets) / BATCH_SIZE,\n",
    "    verbose=2,\n",
    "    callbacks=[early]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2.8094391482216969, 0.37586495535714287]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_cnn_model2.evaluate_generator(\n",
    "    data_generator(test_tweets, batch_size=BATCH_SIZE),\n",
    "    steps=len(test_tweets) / BATCH_SIZE\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"75s - loss: 2.3855 - acc: 0.4368\\n[2.8089022636413574, 0.38840296648550726]\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Featurizing and preparing our data\n",
    "\n",
    "Just like we did when computing word embeddings, we want to featurize our data so we can classify it effectively."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "VOCAB_SIZE = 50000\n",
    "tokenizer = Tokenizer(num_words=VOCAB_SIZE)\n",
    "tokenizer.fit_on_texts(tweets['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training_tokens = tokenizer.texts_to_sequences(train_tweets['text'])\n",
    "test_tokens = tokenizer.texts_to_sequences(test_tweets['text'])\n",
    "max_num_tokens = max(len(x) for x in chain(training_tokens, test_tokens))\n",
    "training_tokens = pad_sequences(training_tokens, maxlen=max_num_tokens)\n",
    "test_tokens = pad_sequences(test_tokens, maxlen=max_num_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training_labels = np.asarray([emoji_to_idx[em] for em in train_tweets['emoji']])\n",
    "test_labels = np.asarray([emoji_to_idx[em] for em in test_tweets['emoji']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_weights(tokenizer):\n",
    "    model = Word2Vec.load('data/twitter_w2v.model')\n",
    "    w2v = np.zeros((tokenizer.num_words, w2v_model.syn0.shape[1]))\n",
    "    for k, v in tokenizer.word_index.items():\n",
    "        if v >= tokenizer.num_words:\n",
    "            continue\n",
    "        if k in w2v_model:\n",
    "            w2v[v] = w2v_model[k]\n",
    "    return w2v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(49874, 50000)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This may take a while to load\n",
    "\n",
    "#w2v = load_weights(tokenizer)\n",
    "\n",
    "#model = Word2Vec.load('data/twitter_w2v.model')\n",
    "w2v = np.zeros((tokenizer.num_words, model.wv.syn0.shape[1]))\n",
    "found = 0\n",
    "for k, v in tokenizer.word_index.items():\n",
    "    if v >= tokenizer.num_words:\n",
    "        continue\n",
    "    if k in model:\n",
    "        w2v[v] = model[k]\n",
    "        found += 1\n",
    "found, tokenizer.num_words\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# World Level CNN\n",
    "\n",
    "As with our previous task, we can try using more powerful models to classify our text.  In this case, the limited training data and text size limit their effectiveness."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "cnn_input (InputLayer)           (None, 54)            0                                            \n",
      "____________________________________________________________________________________________________\n",
      "cnn_embedding (Embedding)        (None, 54, 100)       5000000                                      \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_53 (Conv1D)               (None, 53, 128)       25728                                        \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_56 (Conv1D)               (None, 52, 128)       38528                                        \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_43 (MaxPooling1D)  (None, 26, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_45 (MaxPooling1D)  (None, 26, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_54 (Conv1D)               (None, 25, 256)       65792                                        \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_57 (Conv1D)               (None, 24, 256)       98560                                        \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_44 (MaxPooling1D)  (None, 12, 256)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_46 (MaxPooling1D)  (None, 12, 256)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_55 (Conv1D)               (None, 11, 256)       131328                                       \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_58 (Conv1D)               (None, 10, 256)       196864                                       \n",
      "____________________________________________________________________________________________________\n",
      "global_max_pooling1d_9 (GlobalMa (None, 256)           0                                            \n",
      "____________________________________________________________________________________________________\n",
      "global_max_pooling1d_10 (GlobalM (None, 256)           0                                            \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_8 (Concatenate)      (None, 512)           0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dense_11 (Dense)                 (None, 128)           65664                                        \n",
      "____________________________________________________________________________________________________\n",
      "cnn_predictions (Dense)          (None, 121)           15609                                        \n",
      "====================================================================================================\n",
      "Total params: 5,638,073\n",
      "Trainable params: 5,638,073\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "def create_cnn_model(vocab_size, embedding_size=None, embedding_weights=None, drop_out=0.2):\n",
    "    message = Input(shape=(max_num_tokens,), dtype='int32', name='cnn_input')\n",
    "    \n",
    "    \n",
    "    # The convolution layer in keras does not support masking, so we just allow\n",
    "    # the embedding layer to learn an explicit value.\n",
    "    embedding = Embedding(mask_zero=False, input_dim=vocab_size, \n",
    "                          output_dim=embedding_weights.shape[1], \n",
    "                          weights=[embedding_weights],\n",
    "                          trainable=True,\n",
    "                          name='cnn_embedding')(message)\n",
    "    \n",
    "    global_pools = []\n",
    "    for window in 2, 3:\n",
    "        conv_1x = Conv1D(128, window, activation='relu', padding='valid')(embedding)\n",
    "        max_pool_1x = MaxPooling1D(2)(conv_1x)\n",
    "        conv_2x = Conv1D(256, window, activation='relu', padding='valid')(max_pool_1x)\n",
    "        max_pool_2x = MaxPooling1D(2)(conv_2x)\n",
    "        conv_3x = Conv1D(256, window, activation='relu', padding='valid')(max_pool_2x)\n",
    "\n",
    "        global_pools.append(GlobalMaxPooling1D()(conv_3x))\n",
    "\n",
    "    merged = Concatenate(axis=1)(global_pools)\n",
    "    fc1 = Dense(units=128, activation='elu')(merged)\n",
    "    preds = Dense(units=len(emojis), activation='softmax', name='cnn_predictions')(fc1)\n",
    "    model = Model(\n",
    "        inputs=[message],\n",
    "        outputs=[preds],\n",
    "    )\n",
    "    model.compile(loss='sparse_categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "cnn_model = create_cnn_model(VOCAB_SIZE, embedding_weights=w2v)\n",
    "cnn_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "634695/634695 [==============================] - 112s - loss: 3.2709 - acc: 0.3009   \n",
      "Epoch 2/5\n",
      "634695/634695 [==============================] - 113s - loss: 3.2831 - acc: 0.3012   \n",
      "Epoch 3/5\n",
      "634695/634695 [==============================] - 112s - loss: 3.2785 - acc: 0.3023   \n",
      "Epoch 4/5\n",
      "634695/634695 [==============================] - 112s - loss: 3.2929 - acc: 0.3006   \n",
      "Epoch 5/5\n",
      "634695/634695 [==============================] - 112s - loss: 3.2968 - acc: 0.2990   \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fd4214a8e10>"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn_model.fit(training_tokens, training_labels, epochs=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_lstm_model(vocab_size, embedding_size=None, embedding_weights=None):\n",
    "    message = Input(shape=(None,), dtype='int32', name='lstm_input')\n",
    "    embedding = Embedding(mask_zero=False, input_dim=vocab_size, \n",
    "                          output_dim=embedding_weights.shape[1], \n",
    "                          weights=[embedding_weights],\n",
    "                          trainable=True,\n",
    "                          name='lstm_embedding')(message)\n",
    "\n",
    "    lstm_1 = LSTM(units=128, return_sequences=False)(embedding)\n",
    "    preds = Dense(units=len(emojis), activation='softmax', name='lstm_predictions')(lstm_1)\n",
    "    \n",
    "    model = Model(\n",
    "        inputs=[message],\n",
    "        outputs=[preds],\n",
    "    )\n",
    "    model.compile(loss='sparse_categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_input (InputLayer)      (None, None)              0         \n",
      "_________________________________________________________________\n",
      "lstm_embedding (Embedding)   (None, None, 300)         15000000  \n",
      "_________________________________________________________________\n",
      "lstm_3 (LSTM)                (None, 128)               219648    \n",
      "_________________________________________________________________\n",
      "lstm_predictions (Dense)     (None, 121)               15609     \n",
      "=================================================================\n",
      "Total params: 15,235,257\n",
      "Trainable params: 15,235,257\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "lstm_model = create_lstm_model(VOCAB_SIZE, embedding_weights=w2v)\n",
    "lstm_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/12\n",
      "634695/634695 [==============================] - 40s - loss: 3.3607 - acc: 0.2660    \n",
      "Epoch 2/12\n",
      "634695/634695 [==============================] - 40s - loss: 2.8690 - acc: 0.3510    \n",
      "Epoch 3/12\n",
      "634695/634695 [==============================] - 40s - loss: 2.6476 - acc: 0.3937    \n",
      "Epoch 4/12\n",
      "634695/634695 [==============================] - 40s - loss: 2.4931 - acc: 0.4225    \n",
      "Epoch 5/12\n",
      "634695/634695 [==============================] - 40s - loss: 2.3727 - acc: 0.4445    \n",
      "Epoch 6/12\n",
      "634695/634695 [==============================] - 40s - loss: 2.2696 - acc: 0.4645    \n",
      "Epoch 7/12\n",
      "634695/634695 [==============================] - 40s - loss: 2.1778 - acc: 0.4822    \n",
      "Epoch 8/12\n",
      "634695/634695 [==============================] - 40s - loss: 2.0931 - acc: 0.4989    \n",
      "Epoch 9/12\n",
      "634695/634695 [==============================] - 40s - loss: 2.0135 - acc: 0.5154    \n",
      "Epoch 10/12\n",
      "634695/634695 [==============================] - 40s - loss: 1.9375 - acc: 0.5314    \n",
      "Epoch 11/12\n",
      "634695/634695 [==============================] - 40s - loss: 1.8658 - acc: 0.5476    \n",
      "Epoch 12/12\n",
      "634695/634695 [==============================] - 40s - loss: 1.7976 - acc: 0.5627    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fd4337aa7f0>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_model.fit(training_tokens, training_labels, epochs=12, batch_size=1024, callbacks=[early])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "70522/70522 [==============================] - 24s    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[2.8785435600453315, 0.4029948101352433]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_model.evaluate(test_tokens, test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparing our models\n",
    "\n",
    "Let's compare the predictions from our models on a sample of our data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_char_vectors, _ = next(data_generator(test_tweets, None)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = {\n",
    "    label: [emojis[np.argmax(x)] for x in pred]\n",
    "    for label, pred in (\n",
    "        ('lstm', lstm_model.predict(test_tokens[:100])),\n",
    "        ('char_cnn', char_cnn_model.predict(test_char_vectors[:100])),\n",
    "        ('cnn', cnn_model.predict(test_tokens[:100])),\n",
    "    )\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>true</th>\n",
       "      <th>char_cnn</th>\n",
       "      <th>cnn</th>\n",
       "      <th>lstm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@Gurmeetramrahim @RedFMIndia @rjraunac #8DaysToLionHeart Great</td>\n",
       "      <td>👏</td>\n",
       "      <td>👍</td>\n",
       "      <td>👏</td>\n",
       "      <td>😘</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>@suchsmallgods I can't wait to show him these tweets</td>\n",
       "      <td>😈</td>\n",
       "      <td>😂</td>\n",
       "      <td>❤</td>\n",
       "      <td>😭</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>@Captain_RedWolf I have like 20 set lol WAYYYYYY ahead of you</td>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>.@OtherkinOK were just at @EPfestival, what a set! Next stop is @whelanslive on Friday 11th November 2016.</td>\n",
       "      <td>👌</td>\n",
       "      <td>💪</td>\n",
       "      <td>❤</td>\n",
       "      <td>😊</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>@jochendria: KathNiel with GForce Jorge. #PushAwardsKathNiels</td>\n",
       "      <td>💙</td>\n",
       "      <td>💙</td>\n",
       "      <td>💙</td>\n",
       "      <td>💙</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Okay good</td>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "      <td>❤</td>\n",
       "      <td>😂</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>\"Distraught means to be upset\" \"So that means confused right?\" -@ReevesDakota</td>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "      <td>❤</td>\n",
       "      <td>😭</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>@JennLiri babe wtf call bck I'm tryna listen to this ring tone</td>\n",
       "      <td>🙄</td>\n",
       "      <td>🙄</td>\n",
       "      <td>©</td>\n",
       "      <td>😩</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>does Jen want to be friends? we can so be friends. love you, girl.  #BachelorInParadise</td>\n",
       "      <td>❤</td>\n",
       "      <td>😢</td>\n",
       "      <td>❤</td>\n",
       "      <td>❤</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>@amwalker38: Go Follow these hot accounts @the1stMe420 @DanaDeelish @So_deelish @aka_teemoney38 @CamPromoXXX @SexyLThings @l...</td>\n",
       "      <td>👇</td>\n",
       "      <td>👑</td>\n",
       "      <td>🔥</td>\n",
       "      <td>✨</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>@gspisak: I always made fun of the parents that show up 30+ mins early to pick up their kids today thats me At least I got a...</td>\n",
       "      <td>🙈</td>\n",
       "      <td>🙃</td>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>@ShawnMendes: Toronto Billboard. So cool! @spotify #ShawnXSpotify go find them in your city</td>\n",
       "      <td>😊</td>\n",
       "      <td>😊</td>\n",
       "      <td>😊</td>\n",
       "      <td>😊</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>@kayleeburt77 can I have your number? I seem to have lost mine.</td>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "      <td>🤔</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>@KentMurphy: Tim Tebow hits a dinger on his first pitch seen in professional ball</td>\n",
       "      <td>😳</td>\n",
       "      <td>😳</td>\n",
       "      <td>😳</td>\n",
       "      <td>😳</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>@HailKingSoup: I hate a \"I don't like your girlfriend\" ass female....bitch I like her get the fuck on with your mad ass</td>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "      <td>😭</td>\n",
       "      <td>😂</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>@RoxeteraRibbons Same and I have to figure to prove it</td>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "      <td>😊</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>@theseoulstory: September comebacks: 2PM, SHINee, INFINITE, BTS, Red Velvet, Gain, Song Jieun, Kanto...</td>\n",
       "      <td>🔥</td>\n",
       "      <td>🔥</td>\n",
       "      <td>🔥</td>\n",
       "      <td>🔥</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>@VixenMusicLabel   - Peace &amp; Love</td>\n",
       "      <td>✌</td>\n",
       "      <td>❤</td>\n",
       "      <td>😘</td>\n",
       "      <td>❤</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>@iDrinkGallons  sorry</td>\n",
       "      <td>☹</td>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>@StarYouFollow: 19- Frisson</td>\n",
       "      <td>🙄</td>\n",
       "      <td>👌</td>\n",
       "      <td>😍</td>\n",
       "      <td>✨</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>@RapsDaiIy: Don't sleep on Ugly God</td>\n",
       "      <td>🔥</td>\n",
       "      <td>🔥</td>\n",
       "      <td>🔥</td>\n",
       "      <td>🔥</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>How tf do all my shifts get picked up so quickly?! Wtf</td>\n",
       "      <td>😭</td>\n",
       "      <td>😩</td>\n",
       "      <td>😂</td>\n",
       "      <td>😩</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>@ShadowhuntersTV: #Shadowhunters fans, how many s would YOU give this father-daughter #FlashbackFriday bonding moment betwee...</td>\n",
       "      <td>❤</td>\n",
       "      <td>❤</td>\n",
       "      <td>❤</td>\n",
       "      <td>❤</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>@mbaylisxo: thank god I have a uniform and don't have to worry about what to wear everyday</td>\n",
       "      <td>😅</td>\n",
       "      <td>🙄</td>\n",
       "      <td>❤</td>\n",
       "      <td>😊</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Mood swings like I'm a 13 year old girl</td>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "      <td>🙄</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                            content  \\\n",
       "0                                                                    @Gurmeetramrahim @RedFMIndia @rjraunac #8DaysToLionHeart Great   \n",
       "1                                                                              @suchsmallgods I can't wait to show him these tweets   \n",
       "2                                                                     @Captain_RedWolf I have like 20 set lol WAYYYYYY ahead of you   \n",
       "3                        .@OtherkinOK were just at @EPfestival, what a set! Next stop is @whelanslive on Friday 11th November 2016.   \n",
       "4                                                                     @jochendria: KathNiel with GForce Jorge. #PushAwardsKathNiels   \n",
       "5                                                                                                                         Okay good   \n",
       "6                                                     \"Distraught means to be upset\" \"So that means confused right?\" -@ReevesDakota   \n",
       "7                                                                    @JennLiri babe wtf call bck I'm tryna listen to this ring tone   \n",
       "8                                           does Jen want to be friends? we can so be friends. love you, girl.  #BachelorInParadise   \n",
       "9   @amwalker38: Go Follow these hot accounts @the1stMe420 @DanaDeelish @So_deelish @aka_teemoney38 @CamPromoXXX @SexyLThings @l...   \n",
       "10  @gspisak: I always made fun of the parents that show up 30+ mins early to pick up their kids today thats me At least I got a...   \n",
       "11                                      @ShawnMendes: Toronto Billboard. So cool! @spotify #ShawnXSpotify go find them in your city   \n",
       "12                                                                  @kayleeburt77 can I have your number? I seem to have lost mine.   \n",
       "13                                                @KentMurphy: Tim Tebow hits a dinger on his first pitch seen in professional ball   \n",
       "14          @HailKingSoup: I hate a \"I don't like your girlfriend\" ass female....bitch I like her get the fuck on with your mad ass   \n",
       "15                                                                           @RoxeteraRibbons Same and I have to figure to prove it   \n",
       "16                          @theseoulstory: September comebacks: 2PM, SHINee, INFINITE, BTS, Red Velvet, Gain, Song Jieun, Kanto...   \n",
       "17                                                                                                @VixenMusicLabel   - Peace & Love   \n",
       "18                                                                                                            @iDrinkGallons  sorry   \n",
       "19                                                                                                      @StarYouFollow: 19- Frisson   \n",
       "20                                                                                              @RapsDaiIy: Don't sleep on Ugly God   \n",
       "21                                                                           How tf do all my shifts get picked up so quickly?! Wtf   \n",
       "22  @ShadowhuntersTV: #Shadowhunters fans, how many s would YOU give this father-daughter #FlashbackFriday bonding moment betwee...   \n",
       "23                                       @mbaylisxo: thank god I have a uniform and don't have to worry about what to wear everyday   \n",
       "24                                                                                          Mood swings like I'm a 13 year old girl   \n",
       "\n",
       "   true char_cnn cnn lstm  \n",
       "0     👏        👍   👏    😘  \n",
       "1     😈        😂   ❤    😭  \n",
       "2     😂        😂   😂    😂  \n",
       "3     👌        💪   ❤    😊  \n",
       "4     💙        💙   💙    💙  \n",
       "5     😂        😂   ❤    😂  \n",
       "6     😂        😂   ❤    😭  \n",
       "7     🙄        🙄   ©    😩  \n",
       "8     ❤        😢   ❤    ❤  \n",
       "9     👇        👑   🔥    ✨  \n",
       "10    🙈        🙃   😂    😂  \n",
       "11    😊        😊   😊    😊  \n",
       "12    😂        😂   😂    🤔  \n",
       "13    😳        😳   😳    😳  \n",
       "14    😂        😂   😭    😂  \n",
       "15    😂        😂   😂    😊  \n",
       "16    🔥        🔥   🔥    🔥  \n",
       "17    ✌        ❤   😘    ❤  \n",
       "18    ☹        😂   😂    😂  \n",
       "19    🙄        👌   😍    ✨  \n",
       "20    🔥        🔥   🔥    🔥  \n",
       "21    😭        😩   😂    😩  \n",
       "22    ❤        ❤   ❤    ❤  \n",
       "23    😅        🙄   ❤    😊  \n",
       "24    😂        😂   😂    🙄  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make a dataframe just for test data\n",
    "pd.options.display.max_colwidth = 128\n",
    "test_df = test_tweets[:100].reset_index()\n",
    "eval_df = pd.DataFrame({\n",
    "    'content': test_df['text'],\n",
    "    'true': test_df['emoji'],\n",
    "    **predictions\n",
    "})\n",
    "eval_df[['content', 'true', 'char_cnn', 'cnn', 'lstm']].head(25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Qualitative Evaluation\n",
    "\n",
    "We can examine some of our error cases by hand.  Often, the models tend to agree when they make mistakes, and that the mistakes aren't unreasonable: this task would be challenging even for a human."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>char_cnn</th>\n",
       "      <th>cnn</th>\n",
       "      <th>content</th>\n",
       "      <th>lstm</th>\n",
       "      <th>true</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>👍</td>\n",
       "      <td>👏</td>\n",
       "      <td>@Gurmeetramrahim @RedFMIndia @rjraunac #8DaysToLionHeart Great</td>\n",
       "      <td>😘</td>\n",
       "      <td>👏</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>😂</td>\n",
       "      <td>❤</td>\n",
       "      <td>@suchsmallgods I can't wait to show him these tweets</td>\n",
       "      <td>😭</td>\n",
       "      <td>😈</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>💪</td>\n",
       "      <td>❤</td>\n",
       "      <td>.@OtherkinOK were just at @EPfestival, what a set! Next stop is @whelanslive on Friday 11th November 2016.</td>\n",
       "      <td>😊</td>\n",
       "      <td>👌</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>😂</td>\n",
       "      <td>❤</td>\n",
       "      <td>\"Distraught means to be upset\" \"So that means confused right?\" -@ReevesDakota</td>\n",
       "      <td>😭</td>\n",
       "      <td>😂</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>🙄</td>\n",
       "      <td>©</td>\n",
       "      <td>@JennLiri babe wtf call bck I'm tryna listen to this ring tone</td>\n",
       "      <td>😩</td>\n",
       "      <td>🙄</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>👑</td>\n",
       "      <td>🔥</td>\n",
       "      <td>@amwalker38: Go Follow these hot accounts @the1stMe420 @DanaDeelish @So_deelish @aka_teemoney38 @CamPromoXXX @SexyLThings @l...</td>\n",
       "      <td>✨</td>\n",
       "      <td>👇</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>🙃</td>\n",
       "      <td>😂</td>\n",
       "      <td>@gspisak: I always made fun of the parents that show up 30+ mins early to pick up their kids today thats me At least I got a...</td>\n",
       "      <td>😂</td>\n",
       "      <td>🙈</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "      <td>@kayleeburt77 can I have your number? I seem to have lost mine.</td>\n",
       "      <td>🤔</td>\n",
       "      <td>😂</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>😂</td>\n",
       "      <td>😂</td>\n",
       "      <td>@RoxeteraRibbons Same and I have to figure to prove it</td>\n",
       "      <td>😊</td>\n",
       "      <td>😂</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>❤</td>\n",
       "      <td>😘</td>\n",
       "      <td>@VixenMusicLabel   - Peace &amp; Love</td>\n",
       "      <td>❤</td>\n",
       "      <td>✌</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   char_cnn cnn  \\\n",
       "0         👍   👏   \n",
       "1         😂   ❤   \n",
       "3         💪   ❤   \n",
       "6         😂   ❤   \n",
       "7         🙄   ©   \n",
       "9         👑   🔥   \n",
       "10        🙃   😂   \n",
       "12        😂   😂   \n",
       "15        😂   😂   \n",
       "17        ❤   😘   \n",
       "\n",
       "                                                                                                                            content  \\\n",
       "0                                                                    @Gurmeetramrahim @RedFMIndia @rjraunac #8DaysToLionHeart Great   \n",
       "1                                                                              @suchsmallgods I can't wait to show him these tweets   \n",
       "3                        .@OtherkinOK were just at @EPfestival, what a set! Next stop is @whelanslive on Friday 11th November 2016.   \n",
       "6                                                     \"Distraught means to be upset\" \"So that means confused right?\" -@ReevesDakota   \n",
       "7                                                                    @JennLiri babe wtf call bck I'm tryna listen to this ring tone   \n",
       "9   @amwalker38: Go Follow these hot accounts @the1stMe420 @DanaDeelish @So_deelish @aka_teemoney38 @CamPromoXXX @SexyLThings @l...   \n",
       "10  @gspisak: I always made fun of the parents that show up 30+ mins early to pick up their kids today thats me At least I got a...   \n",
       "12                                                                  @kayleeburt77 can I have your number? I seem to have lost mine.   \n",
       "15                                                                           @RoxeteraRibbons Same and I have to figure to prove it   \n",
       "17                                                                                                @VixenMusicLabel   - Peace & Love   \n",
       "\n",
       "   lstm true  \n",
       "0     😘    👏  \n",
       "1     😭    😈  \n",
       "3     😊    👌  \n",
       "6     😭    😂  \n",
       "7     😩    🙄  \n",
       "9     ✨    👇  \n",
       "10    😂    🙈  \n",
       "12    🤔    😂  \n",
       "15    😊    😂  \n",
       "17    ❤    ✌  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_df[eval_df['lstm'] != eval_df['true']].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 54)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def combined_data_generator(tweets, tokens, batch_size):\n",
    "    tweets = tweets.reset_index()\n",
    "    while True:\n",
    "        batch_idx = random.sample(range(len(tweets)), batch_size)\n",
    "        tweet_batch = tweets.iloc[batch_idx]\n",
    "        token_batch = tokens[batch_idx]\n",
    "        char_vec = np.zeros((batch_size, max_sequence_len, len(chars)))\n",
    "        token_vec = np.zeros((batch_size, max_num_tokens))\n",
    "        y = np.zeros((batch_size,))\n",
    "        for row_idx, (token_row, (_, tweet_row)) in enumerate(zip(token_batch, tweet_batch.iterrows())):\n",
    "            y[row_idx] = emoji_to_idx[tweet_row['emoji']]\n",
    "            for ch_idx, ch in enumerate(tweet_row['text']):\n",
    "                char_vec[row_idx, ch_idx, char_to_idx[ch]] = 1\n",
    "            token_vec[row_idx, :] = token_row\n",
    "        yield {'char_cnn_input': char_vec, 'cnn_input': token_vec, 'lstm_input': token_vec}, y\n",
    "\n",
    "d, y = next(combined_data_generator(train_tweets, training_tokens, 5))\n",
    "d['lstm_input'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "char_cnn_input (InputLayer)      (None, 139, 96)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_35 (Conv1D)               (None, 136, 128)      49280                                        \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_37 (Conv1D)               (None, 135, 128)      61568                                        \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_39 (Conv1D)               (None, 134, 128)      73856                                        \n",
      "____________________________________________________________________________________________________\n",
      "cnn_input (InputLayer)           (None, 54)            0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_29 (MaxPooling1D)  (None, 34, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_31 (MaxPooling1D)  (None, 33, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_33 (MaxPooling1D)  (None, 33, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "cnn_embedding (Embedding)        (None, 54, 300)       15000000                                     \n",
      "____________________________________________________________________________________________________\n",
      "dropout_15 (Dropout)             (None, 34, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_17 (Dropout)             (None, 33, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_19 (Dropout)             (None, 33, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_41 (Conv1D)               (None, 53, 128)       76928                                        \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_44 (Conv1D)               (None, 52, 128)       115328                                       \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_36 (Conv1D)               (None, 31, 256)       131328                                       \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_38 (Conv1D)               (None, 29, 256)       164096                                       \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_40 (Conv1D)               (None, 28, 256)       196864                                       \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_35 (MaxPooling1D)  (None, 26, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_37 (MaxPooling1D)  (None, 26, 128)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_30 (MaxPooling1D)  (None, 7, 256)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_32 (MaxPooling1D)  (None, 7, 256)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_34 (MaxPooling1D)  (None, 7, 256)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_42 (Conv1D)               (None, 25, 256)       65792                                        \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_45 (Conv1D)               (None, 24, 256)       98560                                        \n",
      "____________________________________________________________________________________________________\n",
      "dropout_16 (Dropout)             (None, 7, 256)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_18 (Dropout)             (None, 7, 256)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_20 (Dropout)             (None, 7, 256)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_36 (MaxPooling1D)  (None, 12, 256)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_38 (MaxPooling1D)  (None, 12, 256)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_5 (Concatenate)      (None, 21, 256)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_43 (Conv1D)               (None, 11, 256)       131328                                       \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_46 (Conv1D)               (None, 10, 256)       196864                                       \n",
      "____________________________________________________________________________________________________\n",
      "dropout_21 (Dropout)             (None, 21, 256)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "global_max_pooling1d_5 (GlobalMa (None, 256)           0                                            \n",
      "____________________________________________________________________________________________________\n",
      "global_max_pooling1d_6 (GlobalMa (None, 256)           0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_input (InputLayer)          (None, None)          0                                            \n",
      "____________________________________________________________________________________________________\n",
      "flatten_5 (Flatten)              (None, 5376)          0                                            \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_6 (Concatenate)      (None, 512)           0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_embedding (Embedding)       (None, None, 300)     15000000                                     \n",
      "____________________________________________________________________________________________________\n",
      "dense_8 (Dense)                  (None, 128)           688256                                       \n",
      "____________________________________________________________________________________________________\n",
      "dense_9 (Dense)                  (None, 128)           65664                                        \n",
      "____________________________________________________________________________________________________\n",
      "lstm_3 (LSTM)                    (None, 128)           219648                                       \n",
      "____________________________________________________________________________________________________\n",
      "char_cnn_predictions (Dense)     (None, 121)           15609                                        \n",
      "____________________________________________________________________________________________________\n",
      "cnn_predictions (Dense)          (None, 121)           15609                                        \n",
      "____________________________________________________________________________________________________\n",
      "lstm_predictions (Dense)         (None, 121)           15609                                        \n",
      "____________________________________________________________________________________________________\n",
      "average_4 (Average)              (None, 121)           0                                            \n",
      "====================================================================================================\n",
      "Total params: 32,382,187\n",
      "Trainable params: 32,382,187\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "def prediction_layer(model):\n",
    "    layers = [layer for layer in model.layers if layer.name.endswith('_predictions')]\n",
    "    return layers[0].output\n",
    "\n",
    "def create_ensemble(*models):\n",
    "    inputs = [model.input for model in models]\n",
    "    predictions = [prediction_layer(model) for model in models]\n",
    "    merged = Average()(predictions)\n",
    "    model = Model(\n",
    "        inputs=inputs,\n",
    "        outputs=[merged],\n",
    "    )\n",
    "    model.compile(loss='sparse_categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "\n",
    "ensemble = create_ensemble(char_cnn_model2, cnn_model, lstm_model)\n",
    "ensemble.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "162s - loss: 3.2924 - acc: 0.2773\n",
      "Epoch 2/20\n",
      "243s - loss: 2.8232 - acc: 0.3604\n",
      "Epoch 3/20\n",
      "269s - loss: 2.6129 - acc: 0.4015\n",
      "Epoch 4/20\n",
      "260s - loss: 2.4619 - acc: 0.4321\n",
      "Epoch 5/20\n",
      "437s - loss: 2.3398 - acc: 0.4576\n",
      "Epoch 6/20\n",
      "498s - loss: 2.2380 - acc: 0.4777\n",
      "Epoch 7/20\n",
      "576s - loss: 2.1480 - acc: 0.4968\n",
      "Epoch 8/20\n",
      "193s - loss: 2.0651 - acc: 0.5144\n",
      "Epoch 9/20\n",
      "351s - loss: 1.9845 - acc: 0.5327\n",
      "Epoch 10/20\n",
      "413s - loss: 1.9129 - acc: 0.5482\n",
      "Epoch 11/20\n",
      "757s - loss: 1.8547 - acc: 0.5607\n",
      "Epoch 12/20\n",
      "768s - loss: 1.7917 - acc: 0.5757\n",
      "Epoch 13/20\n",
      "786s - loss: 1.7301 - acc: 0.5886\n",
      "Epoch 14/20\n",
      "784s - loss: 1.6808 - acc: 0.5997\n",
      "Epoch 15/20\n",
      "780s - loss: 1.6333 - acc: 0.6101\n",
      "Epoch 16/20\n",
      "761s - loss: 1.5852 - acc: 0.6204\n",
      "Epoch 17/20\n",
      "751s - loss: 1.5440 - acc: 0.6284\n",
      "Epoch 18/20\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-62-e985aeaa124e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_tweets\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mearly\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m )\n",
      "\u001b[0;32m/home/douwe/proj/notebooks/venv3/lib/python3.5/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     86\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[1;32m     87\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 88\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_support_signature\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minspect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetargspec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/douwe/proj/notebooks/venv3/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_q_size, workers, pickle_safe, initial_epoch)\u001b[0m\n\u001b[1;32m   1871\u001b[0m                     \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'batch'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_index\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1872\u001b[0m                     \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1873\u001b[0;31m                     \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1874\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1875\u001b[0m                     outs = self.train_on_batch(x, y,\n",
      "\u001b[0;32m/home/douwe/proj/notebooks/venv3/lib/python3.5/site-packages/keras/callbacks.py\u001b[0m in \u001b[0;36mon_batch_begin\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m     88\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_delta_ts_batch_begin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mt_before_callbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m         \u001b[0mdelta_t_median\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmedian\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_delta_ts_batch_begin\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     91\u001b[0m         if (self._delta_t_batch > 0. and\n\u001b[1;32m     92\u001b[0m            \u001b[0mdelta_t_median\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0.95\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_delta_t_batch\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/douwe/proj/notebooks/venv3/lib/python3.5/site-packages/numpy/lib/function_base.py\u001b[0m in \u001b[0;36mmedian\u001b[0;34m(a, axis, out, overwrite_input, keepdims)\u001b[0m\n\u001b[1;32m   3942\u001b[0m     \"\"\"\n\u001b[1;32m   3943\u001b[0m     r, k = _ureduce(a, func=_median, axis=axis, out=out,\n\u001b[0;32m-> 3944\u001b[0;31m                     overwrite_input=overwrite_input)\n\u001b[0m\u001b[1;32m   3945\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3946\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/douwe/proj/notebooks/venv3/lib/python3.5/site-packages/numpy/lib/function_base.py\u001b[0m in \u001b[0;36m_ureduce\u001b[0;34m(a, func, **kwargs)\u001b[0m\n\u001b[1;32m   3856\u001b[0m         \u001b[0mkeepdim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3857\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3858\u001b[0;31m     \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3859\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdim\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3860\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/douwe/proj/notebooks/venv3/lib/python3.5/site-packages/numpy/lib/function_base.py\u001b[0m in \u001b[0;36m_median\u001b[0;34m(a, axis, out, overwrite_input)\u001b[0m\n\u001b[1;32m   3975\u001b[0m             \u001b[0mpart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3976\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3977\u001b[0;31m         \u001b[0mpart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpartition\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3978\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3979\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mpart\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/douwe/proj/notebooks/venv3/lib/python3.5/site-packages/numpy/core/fromnumeric.py\u001b[0m in \u001b[0;36mpartition\u001b[0;34m(a, kth, axis, kind, order)\u001b[0m\n\u001b[1;32m    637\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    638\u001b[0m         \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0masanyarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"K\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 639\u001b[0;31m     \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpartition\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkind\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkind\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    640\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    641\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 512\n",
    "ensemble.fit_generator(\n",
    "    combined_data_generator(train_tweets, training_tokens, BATCH_SIZE),\n",
    "    epochs=20,\n",
    "    steps_per_epoch=len(train_tweets) / BATCH_SIZE,\n",
    "    verbose=2,\n",
    "    callbacks=[early]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2.9514826242474541, 0.38115658967391303]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ensemble.evaluate_generator(\n",
    "    combined_data_generator(test_tweets, test_tokens, BATCH_SIZE),\n",
    "    steps=len(test_tweets) / BATCH_SIZE\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "634695"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}