Keras - 08 Text classification (LSTM).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow import keras\n",
    "import pandas as pd\n",
    "from keras import Sequential\n",
    "from keras.layers import LSTM,Dense,Dropout, Embedding, Conv1D, MaxPooling1D, Flatten\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "import os\n",
    "#os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\" \n",
    "#os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"  # use id from $ nvidia-smi\n",
    "\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>label</th>\n",
       "      <th>name</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i went and saw this movie last night after bei...</td>\n",
       "      <td>test</td>\n",
       "      <td>0_10.txt</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>actor turned director bill paxton follows up h...</td>\n",
       "      <td>test</td>\n",
       "      <td>10000_7.txt</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>as a recreational golfer with some knowledge o...</td>\n",
       "      <td>test</td>\n",
       "      <td>10001_9.txt</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>i saw this film in a sneak preview and it is d...</td>\n",
       "      <td>test</td>\n",
       "      <td>10002_8.txt</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bill paxton has taken the true story of the 19...</td>\n",
       "      <td>test</td>\n",
       "      <td>10003_8.txt</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             content label         name  \\\n",
       "0  i went and saw this movie last night after bei...  test     0_10.txt   \n",
       "1  actor turned director bill paxton follows up h...  test  10000_7.txt   \n",
       "2  as a recreational golfer with some knowledge o...  test  10001_9.txt   \n",
       "3  i saw this film in a sneak preview and it is d...  test  10002_8.txt   \n",
       "4  bill paxton has taken the true story of the 19...  test  10003_8.txt   \n",
       "\n",
       "  sentiment  \n",
       "0       pos  \n",
       "1       pos  \n",
       "2       pos  \n",
       "3       pos  \n",
       "4       pos  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def load_imdb(path):\n",
    "    from bs4 import BeautifulSoup\n",
    "    import re, json\n",
    "    import pandas as pd\n",
    "    \n",
    "    def preprocess(text):\n",
    "        text = BeautifulSoup(text.lower(), \"html5lib\").text #removed html tags\n",
    "        text = re.sub(r\"[\\W]+\", \" \", text)\n",
    "        return text\n",
    "    \n",
    "    with open(path, \"r\", encoding=\"utf8\") as f:\n",
    "        comments = pd.DataFrame.from_dict([json.loads(l) for l in f])\n",
    "        comments[\"content\"] = comments[\"content\"].apply(preprocess)\n",
    "        return comments\n",
    "        \n",
    "comments = load_imdb(\"/data/imdb-comments.json\")\n",
    "comments.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>label</th>\n",
       "      <th>name</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25283</th>\n",
       "      <td>this movie is the first of the six infamous gu...</td>\n",
       "      <td>train</td>\n",
       "      <td>10255_9.txt</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19190</th>\n",
       "      <td>the ruins is to turistas as deep impact was to...</td>\n",
       "      <td>test</td>\n",
       "      <td>4772_1.txt</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48651</th>\n",
       "      <td>what a shame it is when a potentially captivat...</td>\n",
       "      <td>train</td>\n",
       "      <td>8788_3.txt</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14310</th>\n",
       "      <td>i got this movie for fifty cents at a going ou...</td>\n",
       "      <td>test</td>\n",
       "      <td>1162_1.txt</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19089</th>\n",
       "      <td>andy milligan the independent movie maker from...</td>\n",
       "      <td>test</td>\n",
       "      <td>4681_1.txt</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2714</th>\n",
       "      <td>from the golden period of british films this h...</td>\n",
       "      <td>test</td>\n",
       "      <td>12443_10.txt</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16736</th>\n",
       "      <td>steven seagal played in many action movies mos...</td>\n",
       "      <td>test</td>\n",
       "      <td>2563_1.txt</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12741</th>\n",
       "      <td>alert this review contains major spoilers do n...</td>\n",
       "      <td>test</td>\n",
       "      <td>10217_4.txt</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35219</th>\n",
       "      <td>spoilers alerthomeward bound the incredible jo...</td>\n",
       "      <td>train</td>\n",
       "      <td>7949_10.txt</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26776</th>\n",
       "      <td>it really is a shame that films like this neve...</td>\n",
       "      <td>train</td>\n",
       "      <td>1159_10.txt</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 content  label          name  \\\n",
       "25283  this movie is the first of the six infamous gu...  train   10255_9.txt   \n",
       "19190  the ruins is to turistas as deep impact was to...   test    4772_1.txt   \n",
       "48651  what a shame it is when a potentially captivat...  train    8788_3.txt   \n",
       "14310  i got this movie for fifty cents at a going ou...   test    1162_1.txt   \n",
       "19089  andy milligan the independent movie maker from...   test    4681_1.txt   \n",
       "2714   from the golden period of british films this h...   test  12443_10.txt   \n",
       "16736  steven seagal played in many action movies mos...   test    2563_1.txt   \n",
       "12741  alert this review contains major spoilers do n...   test   10217_4.txt   \n",
       "35219  spoilers alerthomeward bound the incredible jo...  train   7949_10.txt   \n",
       "26776  it really is a shame that films like this neve...  train   1159_10.txt   \n",
       "\n",
       "      sentiment  \n",
       "25283       pos  \n",
       "19190       neg  \n",
       "48651       neg  \n",
       "14310       neg  \n",
       "19089       neg  \n",
       "2714        pos  \n",
       "16736       neg  \n",
       "12741       neg  \n",
       "35219       pos  \n",
       "26776       pos  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = np.where(comments.sentiment == \"pos\", 1, 0)\n",
    "is_training = comments.label == \"train\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxlen = 1250"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50000"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(comments.content)\n",
    "doc_terms = tokenizer.texts_to_sequences(comments.content)\n",
    "len(doc_terms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'i went and saw this movie last night after being coaxed to by a few friends of mine i ll admit that i was reluctant to see it because from what i knew of ashton kutcher he was only able to do comedy i was wrong kutcher played the character of jake fischer very well and kevin costner played ben randall with such professionalism the sign of a good movie is that it can toy with our emotions this one did exactly that the entire theater which was sold out was overcome by laughter during the first half of the movie and were moved to tears during the second half while exiting the theater i not only saw many women in tears but many full grown men as well trying desperately not to let anyone see them crying this movie was great and i suggest that you go see it before you judge '"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments.content[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[9, 416, 2, 210, 10, 15, 238, 311, 100, 109, 28203, 5, 33, 3, 173, 352, 4, 1758, 9, 232, 975, 11, 9, 13, 5735, 5, 65, 7, 84, 37, 47, 9, 672, 4, 9942, 9315, 24, 13, 62, 477, 5, 78, 201, 9, 13, 356, 9315, 254, 1, 104, 4, 3373, 15901, 53, 69, 2, 1621, 7132, 254, 1155, 8399, 17, 139, 11306, 1, 2016, 4, 3, 49, 15, 6, 11, 7, 50, 2970, 17, 258, 1334, 10, 28, 117, 619, 11, 1, 442, 764, 61, 13, 2986, 43, 13, 3232, 33, 2142, 303, 1, 88, 304, 4, 1, 15, 2, 71, 1649, 5, 1716, 303, 1, 338, 304, 136, 11778, 1, 764, 9, 23, 62, 210, 107, 362, 8, 1716, 18, 107, 364, 2061, 339, 14, 69, 264, 2816, 23, 5, 276, 246, 65, 92, 2488, 10, 15, 13, 80, 2, 9, 1466, 11, 20, 141, 65, 7, 161, 20, 1671]\n"
     ]
    }
   ],
   "source": [
    "print(doc_terms[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39797"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.word_docs[\"i\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"dict_items([('coaxed', 9), ('let', 4066), ('but', 35787), ('in', 44006), ('ashton', 49), ('the', 49560), ('character', 9551), ('with', 34776), ('able', 2361), ('grown', 464), ('it', 44511), ('good', 19000), ('not', 29780), ('emotions', 749), ('only', 16467), ('first', 12611), ('judge', 516), ('our', 3688), ('before', 7146), ('reluctant', 133), ('that', 40473), ('by', 23344), ('he', 20422), ('did', 9445), ('see', 16263), ('women', 2531), ('well', 15001), ('during', 3682), ('movie', 30562), ('a', \""
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(tokenizer.word_docs.items())[:500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "103890"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_dim = len(tokenizer.word_docs.keys()) # Dimension of doc term matrix\n",
    "n_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "words_by_index = dict([(i, word) for word, i in tokenizer.word_index.items()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['i', 'went', 'and', 'saw', 'this', 'movie', 'last', 'night', 'after', 'being', 'coaxed', 'to', 'by', 'a', 'few', 'friends', 'of', 'mine', 'i', 'll', 'admit', 'that', 'i', 'was', 'reluctant', 'to', 'see', 'it', 'because', 'from', 'what', 'i', 'knew', 'of', 'ashton', 'kutcher', 'he', 'was', 'only', 'able', 'to', 'do', 'comedy', 'i', 'was', 'wrong', 'kutcher', 'played', 'the', 'character', 'of', 'jake', 'fischer', 'very', 'well', 'and', 'kevin', 'costner', 'played', 'ben', 'randall', 'with', 'such', 'professionalism', 'the', 'sign', 'of', 'a', 'good', 'movie', 'is', 'that', 'it', 'can', 'toy', 'with', 'our', 'emotions', 'this', 'one', 'did', 'exactly', 'that', 'the', 'entire', 'theater', 'which', 'was', 'sold', 'out', 'was', 'overcome', 'by', 'laughter', 'during', 'the', 'first', 'half', 'of', 'the', 'movie', 'and', 'were', 'moved', 'to', 'tears', 'during', 'the', 'second', 'half', 'while', 'exiting', 'the', 'theater', 'i', 'not', 'only', 'saw', 'many', 'women', 'in', 'tears', 'but', 'many', 'full', 'grown', 'men', 'as', 'well', 'trying', 'desperately', 'not', 'to', 'let', 'anyone', 'see', 'them', 'crying', 'this', 'movie', 'was', 'great', 'and', 'i', 'suggest', 'that', 'you', 'go', 'see', 'it', 'before', 'you', 'judge']\n"
     ]
    }
   ],
   "source": [
    "print([words_by_index[t] for t in doc_terms[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'i'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words_by_index[9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "103890"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_size = len(tokenizer.word_index)\n",
    "vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0xb294c5630>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAD8CAYAAAChHgmuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X2UXPV93/H3d2b2Wc9iJUAPSAaBWRxjYINxcBPHYJCxi9IWjuXGNaUkamJo7NqnKTgtJ+GUc0J6akgbnJhjOMaPAmM7WVMwheCnpraEMNhIgKw1AiMESEJCK620szOz3/5x76yG2Znd0Wp+M7tzP69z9ujOb+69+/tpNfvR7+Hea+6OiIhISKlmV0BERFqfwkZERIJT2IiISHAKGxERCU5hIyIiwSlsREQkOIWNiIgEp7AREZHgFDYiIhJcptkVaISTTjrJV61a1exqiIjMGk8++eQ+d++t1/kSETarVq1iy5Ytza6GiMisYWYv1fN8GkYTEZHgFDYiIhKcwkZERIJT2IiISHAKGxERCU5hIyIiwSlsREQkOIWNNMwPf7mX514danY1RKQJEnFRp8wM19yzGYAX//JDTa6JiDSaejbSELnC2Ph2YcybWBMRaQaFjTTEnkPZ8e0DR0abWBMRaQaFjTTEgeFjAfPGYYWNSNIobKQhhrP58e39wwobkaRR2EhDHBktjG8fzeUn2VNEWpHCRhricEnPZjhbmGRPEWlFChtpiCOj+YrbIpIMChtpiNLeTOmQmogkQ9CwMbO1ZrbdzAbN7MYK73eY2X3x+5vMbFXJezfF5dvN7PKS8hfN7Bkze9rM9PjNWeJoTmEjkmTB7iBgZmngTuADwC7gCTMbcPdnS3a7Djjg7meY2XrgNuAjZtYHrAfOAU4FHjOzM929+Fvqd919X6i6S/1l89FFnemUvWVlmogkQ8iezYXAoLu/4O6jwEZgXdk+64B74+0HgEvMzOLyje6edfedwGB8PpmlcoUx2tJGT3taPRuRBAoZNsuAl0te74rLKu7j7nngILB4imMd+D9m9qSZbQhQbwkglx+jLZ2ipyOjBQIiCRTyRpxWoaz8pljV9pns2IvdfbeZLQEeNbPn3f1HE755FEQbAFauXFl7rSWIXGGM9kyKrvY0w+rZiCROyJ7NLmBFyevlwO5q+5hZBpgP7J/sWHcv/rkH+A5Vhtfc/S5373f3/t7e3hNujJyY0YJHPZv2DEc0ZyOSOCHD5glgjZmtNrN2ogn/gbJ9BoBr4u2rgMfd3ePy9fFqtdXAGmCzmfWY2VwAM+sBLgO2BmyD1EmuMEZ7OkW3ejYiiRRsGM3d82Z2A/AIkAbucfdtZnYLsMXdB4C7ga+Y2SBRj2Z9fOw2M7sfeBbIA9e7e8HMlgLfidYQkAG+7u7fC9UGqZ/xBQIdGfaW3AFaRJIh6MPT3P0h4KGysptLtkeAq6sceytwa1nZC8C59a+phBaFTTxno2E0kcTRHQSkIUbz0ZxNV1uakZyG0USSRmEjDZErjNGWicLmqMJGJHEUNtIQ0QIBo7MtxUhubOoDRKSlKGykIcbnbOKeTbToUESSQmEjDVG8zqajLQ0cu1eaiCSDwkYaoni7mq44bLRIQCRZFDbSENHtaoyu9mLYqGcjkiQKG2mI4pxNZ1v0T04r0kSSRWEjDZEr+FuG0Y7qljUiiaKwkYYYjXs2xQUCI3mFjUiSKGykIYrX2YwvEFDPRiRRFDbSEMXVaJ3q2YgkksJGGiJX8PHb1QAcHdVqNJEkUdhIcO4+PmdTXI2m62xEkkVhI8Hlx6Jb05TO2Wjps0iyKGwkuFwhGjJrS6fobNcdBESSSGEjweXyUc+mLZ2iM6OwEUkihY0EN1rs2WRStKWNdMp0uxqRhFHYSHDFYbT2tGFmdGZSmrMRSRiFjQQ3mj82ZwPQ1a6ndYokjcJGgitdIADQkUlrzkYkYRQ2EtxoYWLPRmEjkiwKGwkuV4ivs8kYAF1taS0QEEkYhY0EVz6M1tmW0iMGRBJGYSPB5fLlYZPWjThFEkZhI8GVz9l0tqXVsxFJGIWNBDc+Z1NcINCWJpvXnI1IkihsJLjxOZt4gYDmbESSJ2jYmNlaM9tuZoNmdmOF9zvM7L74/U1mtqrkvZvi8u1mdnnZcWkze8rMHgxZf6mP8gUCXZqzEUmcYGFjZmngTuCDQB/wUTPrK9vtOuCAu58B3A7cFh/bB6wHzgHWAp+Pz1f0SeC5UHWX+ireQaBdczYiiRWyZ3MhMOjuL7j7KLARWFe2zzrg3nj7AeASM7O4fKO7Z919JzAYnw8zWw58CPhiwLpLHRXnbEoXCGTzY4zFz7kRkdYXMmyWAS+XvN4Vl1Xcx93zwEFg8RTH3gH8KaAZ5lli/EacmWN3EAC0SEAkQUKGjVUoK/+vbLV9Kpab2YeBPe7+5JTf3GyDmW0xsy179+6durYSzLE5m3iBQBw6uhmnSHKEDJtdwIqS18uB3dX2MbMMMB/YP8mxFwNXmtmLRMNy7zezr1b65u5+l7v3u3t/b2/vibdGpq3SvdFAD1ATSZKQYfMEsMbMVptZO9GE/0DZPgPANfH2VcDj7u5x+fp4tdpqYA2w2d1vcvfl7r4qPt/j7v6xgG2QOih9UidEczagno1IkmRCndjd82Z2A/AIkAbucfdtZnYLsMXdB4C7ga+Y2SBRj2Z9fOw2M7sfeBbIA9e7u34zzVK5whjpVPSETjgWNurZiCRHsLABcPeHgIfKym4u2R4Brq5y7K3ArZOc+wfAD+pRTwkrVxgbn68BhY1IEukOAhLcaGFsfAgNoos6AT1mQCRBFDYSXK4wNn5BJxwLG13YKZIcChsJLpf3t/RsOtu09FkkaRQ2ElyuMDZ+E07QnI1IEilsJLjyORuFjUjyKGwkuAlzNu1aICCSNAobCS5XKJuz0e1qRBJHYSPBlV9nk0mnaEubhtFEEkRhI8GN5t86ZwPxM20UNiKJobCR4HKFsfHHCxR1tqU1ZyOSIAobCa58zgbiR0OrZyOSGAobCa58zgaiCzsVNiLJobCR4Mqvs4GoZ6M5G5HkUNhIcOXX2QB0tKV1bzSRBFHYSHDl90aDeM4mrwUCIkmhsJHgyu+NBvGcjXo2IomhsJHgqs3ZjOQVNiJJobCR4CrN2XS1a85GJEkUNhJcpetsOjK6zkYkSRQ2ElRhzCmMVVgg0K6lzyJJorCRoHKFaMVZ+QKB7rY0uYKPvy8irU1hI0GNxmFSPmfT05EBYDibb3idRKTxFDYSVC6+lqZ8GG1OHDaHFTYiiaCwkaByBQcmhk2xZ3NEK9JEEkFhI0GNz9mU3YizuyN6NLR6NiLJoLCRoMbnbDKVh9E0ZyOSDAobCepYz6ZsGK1dYSOSJDWFjZl9y8w+ZGYKJzkuuXzlOZtjPRvN2YgkQa3h8bfAvwZ2mNlfmtnbaznIzNaa2XYzGzSzGyu832Fm98XvbzKzVSXv3RSXbzezy+OyTjPbbGY/N7NtZvYXNdZfmmR0ijmb4VH1bESSoKawcffH3P33gfOBF4FHzez/mdm1ZtZW6RgzSwN3Ah8E+oCPmllf2W7XAQfc/QzgduC2+Ng+YD1wDrAW+Hx8vizwfnc/F3gXsNbMLjqeBktj5apcZ6OlzyLJUvOwmJktBv4t8AfAU8BfE4XPo1UOuRAYdPcX3H0U2AisK9tnHXBvvP0AcImZWVy+0d2z7r4TGAQu9MjheP+2+MtrbYM0Xq7KAoGOTIp0yjRnI5IQtc7ZfBv4MdAN/HN3v9Ld73P3/wDMqXLYMuDlkte74rKK+7h7HjgILJ7sWDNLm9nTwB7gUXffVEsbpDmqLRAwM3ra05qzEUmITI37fdHdHyotMLOOuOfRX+UYq1BW3guptk/VY929ALzLzBYA3zGzd7j71gnf3GwDsAFg5cqVVaoooY3GdxDIpCf+SHs6MurZiCRErcNo/61C2U+mOGYXsKLk9XJgd7V9zCwDzAf213Ksu78J/IBoTmcCd7/L3fvdvb+3t3eKqkoo2ThsOjLpCe/1dGS0QEAkISYNGzM72cwuALrM7DwzOz/+eh/RkNpkngDWmNlqM2snmvAfKNtnALgm3r4KeNzdPS5fH69WWw2sATabWW/co8HMuoBLgedrbq003LGwmfhPracjw2ENo4kkwlTDaJcTLQpYDnyupPwQ8NnJDnT3vJndADwCpIF73H2bmd0CbHH3AeBu4CtmNkjUo1kfH7vNzO4HngXywPXuXjCzU4B745VpKeB+d3/wuFosDVUcRutomxg2czrSGkYTSYhJw8bd7yX65f6v3P1bx3vyeJ7nobKym0u2R4Crqxx7K3BrWdkvgPOOtx7SPOM9m/TEYbTu9gxvHD7S6CqJSBNMGjZm9jF3/yqwysw+Xf6+u3+uwmEi4ybv2WjORiQpphpG64n/rLa8WWRS2Xw0J1N+USdEYXNoRGEjkgRTDaN9If5Tt4WRacnmx2hLG6nUxKXP87qisHF3omt5RaRV1XpR51+Z2TwzazOzfzSzfWb2sdCVk9lvND9WsVcDMK+zjcKY6wFqIglQ63U2l7n7EPBhomtgzgT+U7BaScvI5gt0tE1cHAAwryu6rZ6G0kRaX61hU7zZ5hXAN9x9f6D6SIuZqmcDMDSSa2SVRKQJar1dzXfN7HngKPAJM+sFRsJVS1pFNj9WcSUawNzO6J/f0FGFjUirq/URAzcC7wH63T0HDDPxDs4iE4zmxyrePQCODaOpZyPS+mrt2QCcTXS9TekxX65zfaTFZPNjEx4vUDRvvGejORuRVldT2JjZV4DTgaeB4tIhR2EjU4h6NlMtEFDPRqTV1dqz6Qf64ptkitQsmy9UXSAwPmej1WgiLa/W1WhbgZNDVkRa02QLBDoyaToyKS0QEEmAWns2JwHPmtlmIFssdPcrg9RKWsZkS58hGkrTAgGR1ldr2Px5yEpI64p6NpXnbCBaJKAFAiKtr6awcfcfmtlpwBp3f8zMuomeUSMyKfVsRARqvzfaHwIPAF+Ii5YBfx+qUtI6svlC1aXPEN1FQHM2Iq2v1gUC1wMXA0MA7r4DWBKqUtI6jo4W6G6v3gle0N3GmwobkZZXa9hk3X20+CK+sFPLoGVS7s6RXIGeScJmYXc7+4dHq74vIq2h1rD5oZl9Fugysw8A3wS+G65a0gpGcmO4Q1d79anBRT3tHBrJkyuMNbBmItJotYbNjcBe4Bng3wMPAf8lVKWkNRQf+dzTMUnPpqcdgANH1LsRaWW1rkYbM7O/B/7e3fcGrpO0iCPZ6M5G3ZP1bLrjsBnOsWRuZ0PqJSKNN2nPxiJ/bmb7gOeB7Wa218xubkz1ZDY7kot6NpMtEFjYE90fTfM2Iq1tqmG0TxGtQvtNd1/s7ouAdwMXm9l/DF47mdWGx3s21cNmkYbRRBJhqrD5OPBRd99ZLHD3F4CPxe+JVHV0tPZhNPVsRFrbVGHT5u77ygvjeZu2CvuLjCsuEJj8OpvinI3CRqSVTRU2k/0G0G8HmVSxZ9PTUb1n055JMbcjw34No4m0tKlWo51rZkMVyg3Q0iGZVC09G4iWP2sYTaS1TRo27q6bbcq0HZuzUdiIJF2tF3VOi5mtNbPtZjZoZjdWeL/DzO6L399kZqtK3rspLt9uZpfHZSvM7Ptm9pyZbTOzT4asv5yY4RquswFY1N2m1WgiLS5Y2JhZGrgT+CDQB3zUzPrKdrsOOODuZwC3A7fFx/YB64FzgLXA5+Pz5YHPuPvZwEXA9RXOKTPEkdE8HZkU6ZRNut/CnnYODOtmnCKtLGTP5kJg0N1fiG/iuRFYV7bPOuDeePsB4BIzs7h8o7tn42XXg8CF7v6qu/8MwN0PAc8RPe5AZqCDR3PM75p60eIi3YxTpOWFDJtlwMslr3cxMRjG93H3PHAQWFzLsfGQ23nApjrWWepoaCTHvBrCZmFPO0dzBY6M6omdIq0qZNhUGjspfyxBtX0mPdbM5gDfAj7l7pVWy2FmG8xsi5lt2btXt3Nrhlp7NkvmdgCw91A2dJVEpElChs0uYEXJ6+XA7mr7xM/ImQ/sn+xYM2sjCpqvufu3q31zd7/L3fvdvb+3t/cEmyLTMXQ0z7zOqe/1unRetIr+9SGFjUirChk2TwBrzGy1mbUTTfgPlO0zAFwTb18FPO7uHpevj1errQbWAJvj+Zy7gefc/XMB6y51MDRSW8+mGDZ7Do2ErpKINElNjxiYDnfPm9kNwCNAGrjH3beZ2S3AFncfIAqOr5jZIFGPZn187DYzux94lmgF2vXuXjCz9wL/BnjGzJ6Ov9Vn3f2hUO2Q6Tt4tLY5m+Iwmno2Iq0rWNgAxCHwUFnZzSXbI8DVVY69Fbi1rOz/Unk+R2YYd2foaI55nVOHzYLuNtrTKfVsRFpY0Is6JbkOZ/OMOTUNo5kZvXM72KOejUjLUthIEEMj0TLmeV21dZ6XzutQz0akhSlsJIhDI9EdAebWMIwG0SIBzdmItC6FjQQxnI16NpM9XqDUkrkd7BlSz0akVSlsJIhD8TDanFrDZl4nQyP58TtFi0hrUdhIEMU7Ps+t4aJOOLb8WfM2Iq1JYSNBHM5Gcza1DqMVL+x87aDCRqQVKWwkiMNxz6bWYbTlC7sA2HXgaLA6iUjzKGwkiMPxnE3PFE/pLFq+sJuUwUv7j4Sslog0icJGgjiczdHVliaTru2fWHsmxSnzu3hZYSPSkhQ2EsThbIE5NS4OKFq5qJuX3hgOVCMRaSaFjQRxOJuveb6m6LTF3fx6v+ZsRFqRwkaCGJ5G2KxY1M2+w9nxC0JFpHUobCSIwyN5ejpqWxxQtGpxDwA792koTaTVKGwkiEPZPHM6arsvWtFZJ88F4PnXDoWokog0kcJGgoiG0Y6vZ7P6pB4621I8u3soUK1EpFkUNhLE4Wz+uFejpVPGWUvn8tyrChuRVqOwkSAOT2MYDeDsU+bx3GtDuHuAWolIsyhspO5G82OM5seOexgNoO/Uebx5JMcrb2oJtEgrUdhI3RWXLh/v0meAd61YAMDTL79Z1zqJSHMpbKTuDh/ng9NKvf3keXRkUjz1a4WNSCtR2EjdFR+cVuuzbEq1Z1K8c/l8fvbrA/Wulog0kcJG6m54tDiMdvwLBADOW7mQba8Mkc3rqZ0irUJhI3U3/niBaSwQADhvxQJGC2O63kakhShspO6KczbTGUaDqGcDaN5GpIUobKTuTmSBAMDJ8zs5dX4nT2lFmkjLUNhI3RWH0aaz9LnovJUL+dlLWiQg0ioUNlJ34z2b9hMJmwW88uZR9gyN1KtaItJEQcPGzNaa2XYzGzSzGyu832Fm98XvbzKzVSXv3RSXbzezy0vK7zGzPWa2NWTdZfoOZ/P0tKdJpWza5zhvZXRxp4bSRFpDsLAxszRwJ/BBoA/4qJn1le12HXDA3c8Abgdui4/tA9YD5wBrgc/H5wP4UlwmM9TwNG7CWe6cU+fTljYtEhBpESF7NhcCg+7+gruPAhuBdWX7rAPujbcfAC4xM4vLN7p71t13AoPx+XD3HwH7A9ZbTtChbH7aiwOKOtvS9J06X/M2Ii0iZNgsA14ueb0rLqu4j7vngYPA4hqPlRlqOJtn7gmGDcBFqxfx1MsH9JhokRYQMmwqDdiX3ze+2j61HDv5NzfbYGZbzGzL3r17j+dQOUHRI6FPPGx+58xecgXnJ796ow61EpFmChk2u4AVJa+XA7ur7WNmGWA+0RBZLcdOyt3vcvd+d+/v7e09zqrLiYieZXPiYXPBqoV0taX50Q79Z0FktgsZNk8Aa8xstZm1E034D5TtMwBcE29fBTzu0VOzBoD18Wq11cAaYHPAukodTecpnZV0ZNL81umL+eEvFTYis12wsInnYG4AHgGeA+53921mdouZXRnvdjew2MwGgU8DN8bHbgPuB54Fvgdc7+4FADP7BvAT4Cwz22Vm14Vqg0xPvXo2AO87q5eX3jjCjtcP1eV8ItIc9fmNUIW7PwQ8VFZ2c8n2CHB1lWNvBW6tUP7ROldT6sjdo6XPdQqby845mZsHtvHw1tdYs3RuXc4pIo2nOwhIXWXzY+QKXpcFAgBL53XSf9pCHnrm1bqcT0SaQ2EjdTV8gnd8rmTtO07h+dcOsXPfcN3OKSKNpbCRuqrHfdHKrX3HyQA8vFW9G5HZSmEjdVV8JHQ9VqMVLVvQxbkrFvDwM6/V7Zwi0lgKG6mr8WG0Os3ZFF3xjpN55pWDvLz/SF3PKyKNobCRujrRB6dV88F3nALAd39xXNf2isgMobCRuiqGTT2H0QBWLu7morct4ms//TX5wlhdzy0i4SlspK7Gw6bOPRuAay9ezStvHuV72zR3IzLbKGykrurxSOhqLj17KWcsmcMdj+2gMHZc92UVkSZT2EhdDWfzmEF3e3rqnY9TOmV8+gNnMrjnMAM/f6Xu5xeRcBQ2UleHsnnmtGeInoFXf2vPOZm+U+Zxx2M7yGnuRmTWUNhIXR0ayTOvqy3Y+VMp4zOXnclLbxzhgSd3Bfs+IlJfChupq4NHc3W9VU0l73/7Es5buYDbH/0lQyO5oN9LROpDYSN1NXQ0F7RnA2Bm/MWV57DvcJbbHn4+6PcSkfpQ2EhdDY3kmdcZNmwA3rl8AddevJqvbfo1m17QY6NFZjqFjdTV0NEc8wP3bIo+c9mZrFzUzWe++XMOaThNZEZT2EhdRcNoYedsirrbM9z+kXPZ/eZRbvnusw35niIyPQobqZvCmHMo25hhtKILTlvEJ953Bt98chff26o7C4jMVAobqZviUFboBQLl/uSSNfzGsvl8+v6nNX8jMkMpbKRuho5Gt6pp1JxNUXsmxd3X9HPqgi4+fs9mHtSdoUVmHIWN1M2BI6NA48MGYMm8TjZuuIh3LJvPDV9/ijse+yXuun+ayEyhsJG62XMoC8CSuR1N+f4nzeng63/4bv7l+cu447EdXP/1n2mVmsgMobCRutlzaASApfM6m1aHjkya/3H1uXz2irfzyLbXufJv/olndw81rT4iElHYSN28PpTFDE6a097UepgZG377dDZuuIgjo3l+7/P/xDc2/1rDaiJNpLCRutl7aITFPR1k0jPjn9VvrlrE//6Tf8a7Vy/ipm8/wx999Ul27htudrVEEmlm/FaQlvD6ULZp8zXVnDSngy9deyF/uvYsfrxjH5d+7of88Vef5MFf7GY4fqqoiITXmEu9JRF2v3mUUxd0NbsaE6RTxifedwZXX7CCv/vhr/iHp3fz8NbX6Mik+O0ze1l7zsn81hmLOWX+zKu7SKtQ2EhdFMacnfuGee8ZJzW7KlX1zu3gv364j89ecTabd+7nkW2v8b2tr/Hos68DcOr8Ts4/bSG/c2YvH+hbyoLuyeeejozmeWbXQc5cOpeFPc2dpxKZ6YKGjZmtBf4aSANfdPe/LHu/A/gycAHwBvARd38xfu8m4DqgAPyJuz9SyzmlOV58Y5hsfowzl85tdlWmlE4Z7zl9Me85fTE3f7iPbbuH2PLSfp586QBPvLifB3/xKumU0X/aQt79tsVccNpCFve005ZOURhzRgtjPPbs63x100u8eSRHd3uav/3YBfzOmb3NbprIjBUsbMwsDdwJfADYBTxhZgPuXnrHxOuAA+5+hpmtB24DPmJmfcB64BzgVOAxMzszPmaqc0oTbN65H4DzT1vQ5Jocn1TK+I3l8/mN5fO59uLVuDvPvHKQh7e+xo937OVvHt/BWIVFbGbwgbOXsu5dy7jz+4Ns+PIWvnTthbzn9MWNb4TILBCyZ3MhMOjuLwCY2UZgHVAaDOuAP4+3HwD+xqKH168DNrp7FthpZoPx+ajhnNJgucIYG594mdMWd3N675xmV+eEmBnvXL6Ady5fwH9e+3YOjeR45pWDDGcLjObHSKcgZcY5y+azLJ6fes/pi/nIF37CH9z7BP/96nN5/9uX0NmWbnJLRGaWkGGzDHi55PUu4N3V9nH3vJkdBBbH5T8tO3ZZvD3VOevmw//rx4zkxsZfl1+nUfGqDZ/0ZcVrPSbuU+m0Puk+tVxCUkv9J5y3wl7l+xwZLXA4m+f2j5xL9H+F1jG3s43fOn3yeahFPe189Q/ezTX3bOYTX/sZAG1pI50yDMMMin8rZhZtx2Vmx94vvmfxDsW/ymJZ6bmKf89mE9+r9n2ktVX6CS/sbuf+P3pPw+tSSciwqdT28t9c1fapVl5pqXbFX7NmtgHYALBy5crqtZzEGb1zyBXKTm+Tvix+70n3qfS5n7jPxJ0mlEyoS4Vjpqhv5brY1PuUlGVSKd5/9hJ+96wlE3dMiKXzOhm44b386Jd7ef61IQ5nC4y54+7j4exEQe2UlLlPKC++Lh7lXvl9J3rhJedh/L2SMl3L2vIq/acQaOjjPqYSMmx2AStKXi8Hym/HW9xnl5llgPnA/imOneqcALj7XcBdAP39/dP6uN2x/rzpHCYJ1Z5JcWnfUi7tW9rsqojMOCEv6nwCWGNmq82snWjCf6BsnwHgmnj7KuBxj8Z6BoD1ZtZhZquBNcDmGs8pIiIzTLCeTTwHcwPwCNEy5XvcfZuZ3QJscfcB4G7gK/ECgP1E4UG83/1EE/954Hp3LwBUOmeoNoiISH1YEm5O2N/f71u2bGl2NUREZg0ze9Ld++t1Pt0bTUREglPYiIhIcAobEREJTmEjIiLBKWxERCS4RKxGM7O9wEsN/rYnAfsa/D1ngiS2W21OhqS1+TTgz+IL5E9YIsKmGcxsSz2XDc4WSWy32pwMavOJ0TCaiIgEp7AREZHgFDbh1GWccxZKYrvV5mRQm0+A5mxERCQ49WxERCQ4hc00mdnVZrbNzMbMrL/svZvMbNDMtpvZ5SXla+OyQTO7saR8tZltMrMdZnZf/PiEWaVa22YjM7vHzPaY2daSskVm9mj8M3rUzBbG5WZm/zNu9y/M7PySY66J999hZtdU+l4zhZmtMLPvm9lz8b/rT8blLdtuM+s0s81m9vO4zX8Rl1f8PMaPPLkvbvMmM1tVcq6Kn/mZyszSZvaUmT0Yvw7fZh9/mqC+jucLOBs4C/gB0F9S3gf8HOgAVgO/InocQjrefhvQHu/TFx9zP7A+3v474I+b3b7j/Ltj+dbiAAADn0lEQVSo2rbZ+AX8NnA+sLWk7K+AG+PtG4Hb4u0rgIeJHoJ6EbApLl8EvBD/uTDeXtjstk3S5lOA8+PtucAv43/LLdvuuO5z4u02YFPcloqfR+ATwN/F2+uB++Ltip/5ZrdvirZ/Gvg68GD8Onib1bOZJnd/zt23V3hrHbDR3bPuvhMYBC6Mvwbd/QV3HwU2Aussev7z+4EH4uPvBX4vfAvqqmLbmlynaXP3HxE9X6nUOqKfDbz1Z7QO+LJHfgosMLNTgMuBR919v7sfAB4F1oav/fS4+6vu/rN4+xDwHLCMFm53XPfD8cu2+Mup/nks/bt4ALgk/vxW+8zPSGa2HPgQ8MX49WS/g+rWZoVN/S0DXi55vSsuq1a+GHjT3fNl5bNJtba1kqXu/ipEv5iBJXH58f68Z7x4qOQ8ov/pt3S74+Gkp4E9RMH4K6p/HsfbFr9/kOjzO6vaDNwB/CkwFr+e7HdQ3doc7EmdrcDMHgNOrvDWn7n7P1Q7rEKZUznYfZL9Z5NWaMN0VWv7rPw7MbM5wLeAT7n7UPSf2Mq7Viibde326AnA7zKzBcB3iIbHJ+wW/znr22xmHwb2uPuTZva+YnGFXeveZoXNJNz90mkctgtYUfJ6ObA73q5Uvo9oCCIT/8+hdP/ZYrI2t4rXzewUd381Hi7aE5dXa/su4H1l5T9oQD2nzczaiILma+7+7bi45dsN4O5vmtkPiOZsqn0ei23eZWYZYD7RcOts+vd/MXClmV0BdALziHo6wdusYbT6GwDWx6s4VgNrgM3AE8CaeNVHO9Fk24BHs23fB66Kj78GqNZrmqkqtq3Jdaq3AaKfDbz1ZzQAfDxenXURcDAebnoEuMzMFsYruC6Ly2akeBz+buA5d/9cyVst224z6417NJhZF3Ap0VxVtc9j6d/FVcDj8ee32md+xnH3m9x9ubuvIvqcPu7uv08j2tzsVRGz9Qv4F0TpngVeBx4pee/PiMZ+twMfLCm/gmiVz6+IhuKK5W+Lf1CDwDeBjma3bxp/HxXbNhu/gG8ArwK5+Gd8HdE49T8CO+I/F8X7GnBn3O5neOvKxH8X/0wHgWub3a4p2vxeomGQXwBPx19XtHK7gXcCT8Vt3grcHJdX/DwS9QS+GZdvBt5Wcq6Kn/mZ/EXUAy2uRgveZt1BQEREgtMwmoiIBKewERGR4BQ2IiISnMJGRESCU9iIiEhwChsREQlOYSMiIsEpbEREJLj/D7biSYYfellrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.Series([len(r) for r in doc_terms]).plot.kde()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   0,    0,    0, ...,  161,   20, 1671], dtype=int32)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_terms_padded = pad_sequences(doc_terms, maxlen=maxlen)\n",
    "doc_terms_padded[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = doc_terms_padded[is_training]\n",
    "y_train = y[is_training]\n",
    "x_test = doc_terms_padded[~is_training]\n",
    "y_test = y[~is_training]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /anaconda3/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 1250, 10)          1038910   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 1250, 10)          0         \n",
      "_________________________________________________________________\n",
      "conv1d_1 (Conv1D)            (None, 1246, 64)          3264      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1 (None, 311, 64)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_2 (Conv1D)            (None, 309, 128)          24704     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1 (None, 77, 128)           0         \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 128)               131584    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 400)               51600     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 401       \n",
      "=================================================================\n",
      "Total params: 1,250,463\n",
      "Trainable params: 1,250,463\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:From /anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/100\n",
      "25000/25000 [==============================] - 83s 3ms/step - loss: 0.4634 - acc: 0.7496 - val_loss: 0.3175 - val_acc: 0.8632\n",
      "Epoch 2/100\n",
      "25000/25000 [==============================] - 83s 3ms/step - loss: 0.2359 - acc: 0.9092 - val_loss: 0.3148 - val_acc: 0.8673\n",
      "Epoch 3/100\n",
      "25000/25000 [==============================] - 83s 3ms/step - loss: 0.1536 - acc: 0.9439 - val_loss: 0.3851 - val_acc: 0.8543\n",
      "Epoch 4/100\n",
      "25000/25000 [==============================] - 84s 3ms/step - loss: 0.1122 - acc: 0.9598 - val_loss: 0.4364 - val_acc: 0.8394\n",
      "Epoch 5/100\n",
      "25000/25000 [==============================] - 84s 3ms/step - loss: 0.0799 - acc: 0.9721 - val_loss: 0.4780 - val_acc: 0.8481\n",
      "Epoch 6/100\n",
      "25000/25000 [==============================] - 84s 3ms/step - loss: 0.0687 - acc: 0.9761 - val_loss: 0.5431 - val_acc: 0.8204\n",
      "Epoch 7/100\n",
      "25000/25000 [==============================] - 84s 3ms/step - loss: 0.0511 - acc: 0.9813 - val_loss: 0.5588 - val_acc: 0.8357\n",
      "Epoch 8/100\n",
      "25000/25000 [==============================] - 85s 3ms/step - loss: 0.0472 - acc: 0.9824 - val_loss: 0.6069 - val_acc: 0.8391\n",
      "Epoch 9/100\n",
      "25000/25000 [==============================] - 84s 3ms/step - loss: 0.0403 - acc: 0.9853 - val_loss: 0.6040 - val_acc: 0.8334\n",
      "Epoch 10/100\n",
      "25000/25000 [==============================] - 84s 3ms/step - loss: 0.0398 - acc: 0.9852 - val_loss: 0.6148 - val_acc: 0.8409\n",
      "Epoch 11/100\n",
      "25000/25000 [==============================] - 84s 3ms/step - loss: 0.0522 - acc: 0.9817 - val_loss: 0.5729 - val_acc: 0.8310\n",
      "Epoch 12/100\n",
      "25000/25000 [==============================] - 84s 3ms/step - loss: 0.0401 - acc: 0.9852 - val_loss: 0.6912 - val_acc: 0.8380\n",
      "Epoch 13/100\n",
      "25000/25000 [==============================] - 84s 3ms/step - loss: 0.0273 - acc: 0.9898 - val_loss: 0.6004 - val_acc: 0.8397\n",
      "Epoch 14/100\n",
      "25000/25000 [==============================] - 84s 3ms/step - loss: 0.0296 - acc: 0.9901 - val_loss: 0.7360 - val_acc: 0.8312\n",
      "Epoch 15/100\n",
      "25000/25000 [==============================] - 85s 3ms/step - loss: 0.0242 - acc: 0.9923 - val_loss: 0.6262 - val_acc: 0.8336\n",
      "Epoch 16/100\n",
      "25000/25000 [==============================] - 85s 3ms/step - loss: 0.0223 - acc: 0.9918 - val_loss: 0.6992 - val_acc: 0.8360\n",
      "Epoch 17/100\n",
      "25000/25000 [==============================] - 85s 3ms/step - loss: 0.0228 - acc: 0.9924 - val_loss: 0.7315 - val_acc: 0.8343\n",
      "Epoch 18/100\n",
      "25000/25000 [==============================] - 87s 3ms/step - loss: 0.0212 - acc: 0.9930 - val_loss: 0.6191 - val_acc: 0.8184\n",
      "Epoch 19/100\n",
      "25000/25000 [==============================] - 85s 3ms/step - loss: 0.0191 - acc: 0.9929 - val_loss: 0.7274 - val_acc: 0.8322\n",
      "Epoch 20/100\n",
      "25000/25000 [==============================] - 87s 3ms/step - loss: 0.0210 - acc: 0.9920 - val_loss: 0.8206 - val_acc: 0.8112\n",
      "Epoch 21/100\n",
      "25000/25000 [==============================] - 88s 4ms/step - loss: 0.0181 - acc: 0.9934 - val_loss: 0.7269 - val_acc: 0.8308\n",
      "Epoch 22/100\n",
      "25000/25000 [==============================] - 88s 4ms/step - loss: 0.0172 - acc: 0.9942 - val_loss: 0.6016 - val_acc: 0.8359\n",
      "Epoch 23/100\n",
      "25000/25000 [==============================] - 85s 3ms/step - loss: 0.0172 - acc: 0.9940 - val_loss: 0.8079 - val_acc: 0.8064\n",
      "Epoch 24/100\n",
      "25000/25000 [==============================] - 82s 3ms/step - loss: 0.0152 - acc: 0.9952 - val_loss: 0.9236 - val_acc: 0.8288\n",
      "Epoch 25/100\n",
      "25000/25000 [==============================] - 88s 4ms/step - loss: 0.0163 - acc: 0.9943 - val_loss: 0.8518 - val_acc: 0.8270\n",
      "Epoch 26/100\n",
      "25000/25000 [==============================] - 80s 3ms/step - loss: 0.0173 - acc: 0.9943 - val_loss: 1.0413 - val_acc: 0.7962\n",
      "Epoch 27/100\n",
      "25000/25000 [==============================] - 82s 3ms/step - loss: 0.0153 - acc: 0.9946 - val_loss: 0.8499 - val_acc: 0.8333\n",
      "Epoch 28/100\n",
      "25000/25000 [==============================] - 82s 3ms/step - loss: 0.0141 - acc: 0.9955 - val_loss: 0.7715 - val_acc: 0.8269\n",
      "Epoch 29/100\n",
      "25000/25000 [==============================] - 81s 3ms/step - loss: 0.0109 - acc: 0.9962 - val_loss: 1.0707 - val_acc: 0.8280\n",
      "Epoch 30/100\n",
      "25000/25000 [==============================] - 81s 3ms/step - loss: 0.0111 - acc: 0.9958 - val_loss: 0.7635 - val_acc: 0.8298\n",
      "Epoch 31/100\n",
      "25000/25000 [==============================] - 82s 3ms/step - loss: 0.0139 - acc: 0.9949 - val_loss: 0.7900 - val_acc: 0.8266\n",
      "Epoch 32/100\n",
      " 2560/25000 [==>...........................] - ETA: 1:02 - loss: 0.0132 - acc: 0.9961"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-25-d715b45f7cd8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'binary_crossentropy'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'adam'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msummary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m64\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/anaconda3/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1040\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/anaconda3/lib/python3.7/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2713\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2715\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2716\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2717\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2674\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2675\u001b[0;31m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2676\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1437\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1438\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1439\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1440\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "tf.set_random_seed(1)\n",
    "np.random.seed(1)\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim = n_dim + 1, output_dim = 10, input_length=maxlen))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Conv1D(64, 5, activation='elu'))\n",
    "model.add(MaxPooling1D(pool_size=4))\n",
    "model.add(Conv1D(128, 3, activation='elu'))\n",
    "model.add(MaxPooling1D(pool_size=4))\n",
    "model.add(LSTM(128))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(400, activation='elu'))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.summary()\n",
    "model.fit(x_train, y_train, validation_data=(x_test, y_test), batch_size=64, epochs=100)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}