NLP/IMDB.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment analysis using IMDB dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /Users/raymondyuan/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from glob import glob\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import svm\n",
    "import zipfile\n",
    "from tqdm import tqdm\n",
    "from nltk.tokenize import word_tokenize\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import re\n",
    "from scipy import sparse\n",
    "import nltk\n",
    "\n",
    "# Download any necessary nltk files for nlp\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "zip_file_path = './imdb_dataset.zip'\n",
    "extract_dir = './'\n",
    "data_dir = 'imdb_dataset'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Extract all the files \n",
    "zip_ref = zipfile.ZipFile(zip_file_path, 'r')\n",
    "zip_ref.extractall(extract_dir)\n",
    "zip_ref.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's begin by reading in all of our text files. We'll create their label according to their sentiment, either positive or negative. In addition we'll preprocess all the nexts by removing all non-alpha numeric characters. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 12500/12500 [00:01<00:00, 9482.37it/s] \n",
      "100%|██████████| 12500/12500 [00:02<00:00, 4411.21it/s]\n",
      "100%|██████████| 12500/12500 [00:02<00:00, 4378.72it/s]\n",
      "100%|██████████| 12500/12500 [00:02<00:00, 4344.35it/s]\n"
     ]
    }
   ],
   "source": [
    "# Regex to remove all Non-Alpha Numeric \n",
    "SPECIAL_CHARS = re.compile(r'([^a-z\\d!?.\\s])', re.IGNORECASE)\n",
    "\n",
    "def read_texts(glob_to_texts):\n",
    "    texts = []\n",
    "    labels = []\n",
    "    label = int(\"pos\" in glob_to_texts)\n",
    "    for text_name in tqdm(glob(glob_to_texts)):\n",
    "        with open(text_name, 'r') as text:\n",
    "            # Removing all non-alphanumeric\n",
    "            filter_text = SPECIAL_CHARS.sub('',  text.read())\n",
    "            texts.append(filter_text)\n",
    "            labels.append(label)\n",
    "    return texts, labels\n",
    "\n",
    "# Get all training data\n",
    "train_pos_data = read_texts(os.path.join(data_dir, \"train/pos/*.txt\"))\n",
    "train_neg_data = read_texts(os.path.join(data_dir, \"train/neg/*.txt\"))\n",
    "\n",
    "# Get all test data\n",
    "test_pos_data = read_texts(os.path.join(data_dir, \"test/pos/*.txt\"))\n",
    "test_neg_data = read_texts(os.path.join(data_dir, \"test/neg/*.txt\"))\n",
    "\n",
    "train_texts = train_pos_data[0] + train_neg_data[0]\n",
    "train_labels = train_pos_data[1] + train_neg_data[1]\n",
    "\n",
    "test_texts = test_pos_data[0] + test_neg_data[0]\n",
    "test_labels = test_pos_data[1] + test_neg_data[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split the data into training and validation sets. We'll create a validation test set with 10% of the data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_texts, val_texts, train_labels, val_labels = train_test_split(train_texts, train_labels, test_size=0.1,\n",
    "                                                                    random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vectorization\n",
    "In order to extract information from text, we'll vectorize our word sequences. In other words, we'll transform our sentences into numerical features. There are many vectorization or embedding techniques such as Bag of Words, Pre-Trained word embeddings, but in our case we'll be using **TF-IDF**.\n",
    "\n",
    "TF-IDF stands for \"Term Frequency, Inverse Document Frequency\". It's a technique that converts words into an importance score of each word in the document based on how they appear accros multiple documents. Intuitively, the TF-IDF score of a word is high when it is frequently found in a document. However, if the word appears in many documents, this word is not a unique identifier, and as such, will have a lower score. For example, common words such as \"the\" and \"and\" will have low score since they appear in many documents. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vec = TfidfVectorizer(ngram_range=(1, 2), tokenizer=word_tokenize,\n",
    "                      min_df=3, max_df=0.9, strip_accents='unicode', use_idf=1,\n",
    "                      smooth_idf=1, sublinear_tf=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We fit our vectorizer to our entire corpus of words, which includes the training, validation, and test sets. Once fitted, we'll transform each subset of the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created Vectorizer TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
      "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
      "        lowercase=True, max_df=0.9, max_features=None, min_df=3,\n",
      "        ngram_range=(1, 2), norm='l2', preprocessor=None, smooth_idf=1,\n",
      "        stop_words=None, strip_accents='unicode', sublinear_tf=1,\n",
      "        token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
      "        tokenizer=<function word_tokenize at 0x113a678c8>, use_idf=1,\n",
      "        vocabulary=None)\n",
      "Fitting to all docs...\n",
      "Transforming to train docs...\n",
      "Transforming to val docs...\n",
      "Transforming to test docs...\n"
     ]
    }
   ],
   "source": [
    "print(\"Created Vectorizer %s\" % vec)\n",
    "print(\"Fitting to all docs...\")\n",
    "vec.fit(train_texts + val_texts + test_texts)\n",
    "print(\"Transforming train docs...\")\n",
    "trn_term_doc = vec.transform(train_texts)\n",
    "print(\"Transforming val docs...\")\n",
    "val_term_doc = vec.transform(val_texts)\n",
    "print(\"Transforming test docs...\")\n",
    "test_term_doc = vec.transform(test_texts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model\n",
    "If you're unfamiliar or want a refresher on SVM's you should check out our [CV tutorial](https://github.com/abhmul/DataScienceTrack/blob/master/CV/Tutorial.ipynb)!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, ClassifierMixin\n",
    "from sklearn.utils.validation import check_X_y, check_is_fitted\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "class NbSvmClassifier(BaseEstimator, ClassifierMixin):\n",
    "    def __init__(self, C=1.0, dual='auto', verbose=0):\n",
    "        self.C = C\n",
    "        self.dual = dual\n",
    "        self.verbose = verbose\n",
    "        self._clf = None\n",
    "        print(\"Creating model with C=%s\" % C)\n",
    "\n",
    "    def predict(self, x):\n",
    "        # Verify that model has been fit\n",
    "        check_is_fitted(self, ['_r', '_clf'])\n",
    "        return self._clf.predict(x.multiply(self._r))\n",
    "    \n",
    "    def score(self, x, y):\n",
    "        check_is_fitted(self, ['_r', '_clf'])\n",
    "        return self._clf.score(x.multiply(self._r), y)\n",
    "        \n",
    "    def fit(self, x, y):\n",
    "        # Check that X and y have correct shape\n",
    "        x, y = check_X_y(x, y, accept_sparse=True)\n",
    "\n",
    "        def pr(x, y_i, y):\n",
    "            p = x[y == y_i].sum(0)\n",
    "            return (p + 1) / ((y == y_i).sum() + 1)\n",
    "\n",
    "        self._r = sparse.csr_matrix(np.log(pr(x, 1, y) / pr(x, 0, y)))\n",
    "        x_nb = x.multiply(self._r)\n",
    "        if self.dual == 'auto':\n",
    "            self.dual = x_nb.shape[0] <= x_nb.shape[1]\n",
    "        self._clf = LinearSVC(C=self.C, dual=self.dual, verbose=self.verbose)\n",
    "        self._clf.fit(x_nb, y)\n",
    "        return self"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finding optimal parameters\n",
    "We'll perform a grid search across the C parameter to find the optimal parameter for our dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting with C=0.01\n",
      "Creating model with C=0.01\n",
      "Model had val score of 0.8496\n",
      "New maximum score improved from -inf to 0.8496\n",
      "Fitting with C=0.1\n",
      "Creating model with C=0.1\n",
      "Model had val score of 0.8844\n",
      "New maximum score improved from 0.8496 to 0.8844\n",
      "Fitting with C=1.0\n",
      "Creating model with C=1.0\n",
      "Model had val score of 0.9112\n",
      "New maximum score improved from 0.8844 to 0.9112\n",
      "Fitting with C=10.0\n",
      "Creating model with C=10.0\n",
      "Model had val score of 0.9132\n",
      "New maximum score improved from 0.9112 to 0.9132\n",
      "Fitting with C=100.0\n",
      "Creating model with C=100.0\n",
      "Model had val score of 0.9064\n",
      "Best score with C=10.0 is 0.9132\n"
     ]
    }
   ],
   "source": [
    "# Search for the appropriate C\n",
    "Cs = [1e-2, 1e-1, 1e0, 1e1, 1e2]\n",
    "\n",
    "best_model = None\n",
    "best_val = -float(\"inf\")\n",
    "best_C = None\n",
    "for C in Cs:\n",
    "    print(\"Fitting with C={}\".format(C))\n",
    "    model = NbSvmClassifier(C=C, verbose=0).fit(trn_term_doc, train_labels)\n",
    "    # Evaluate the model\n",
    "    val_preds = model.predict(val_term_doc)\n",
    "    score = np.mean(val_labels == val_preds)\n",
    "\n",
    "    print(\"Model had val score of %s\" % score)\n",
    "    if score > best_val:\n",
    "        print(\"New maximum score improved from {} to {}\".format(best_val, score))\n",
    "        best_model = model\n",
    "        best_val = score\n",
    "        best_C = C\n",
    "score = best_val\n",
    "print(\"Best score with C={} is {}\".format(best_C, score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.90932"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model.score(test_term_doc, test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Takeaways\n",
    "From this tutorial, we learned how to work with text data and use a basic embedding. In addition, we realize that deep learning isn't always the way to go! We trained a fast and powerful linear model that achieved ~**91**%!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sample Texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Positive Sentiment example\n",
      "This first two seasons of this comedy series were very strange and they werent very funny and had a drama element where Bill the mother was struggling with all the usual problems in life but that element was a bit depressing and didnt mix well with th comedy elements which is probably why it was dropped. After that it soon became one of the funniest comedy series the BBC have ever made! The chemistry between Bill and Bens characters were very funny and there was always so many brilliant and memorable sketches in each series. The Christmas specials were hilarious and a real treat for Christmas. br br The show came to a stop when the main actor Gary Olsen playing Bill passed away which was very sad because he was a brilliant actor in films such as Up n Under and a very funny man RIPbr br This underrated show has sadly disappeared from our television screens and doesnt to be repeated that often  Though it does appear on UKTV Gold once in a while but it should be repeated on BBC one or two to show this brilliant Comedy to a new audience\n",
      "---------------------------\n",
      "Negative Sentiment example\n",
      "WriterDirector Bart Sibrel bases his work here around a can of film that he says was mistakenly sent to him by NASA. He says it shows the astronauts faking the television footage of their trip to the moon by employing camera tricks. The astronauts were in low Earth orbit all the time and editors on the ground composed this raw footage into just a few seconds of finished film.br br Unfortunately Sibrels research is so slipshod that he doesnt realize his backstage footage is really taken in large part from the 30minute live telecast also on that reel that was seen by millions not hidden away in NASA vaults as he implies. And we have to wonder why Sibrel puts his own conspiratorial narration over the astronauts audio in the footage because hearing the astronauts in their own words clearly spells out that the astronauts were just testing the camera not faking footage.br br Finally anyone can see the raw footage for themselves without having to buy Sibrels hackedup version of it. He shows you more of the Zapruder film of JFKs assassination than of his smoking gun. Sibrel thinks hes the only one whos seen it. Whats more revealing is the clips from that raw footage that Sibrel chose NOT to use such as those clearly showing the appropriately distant Earth being eclipsed by the window frames and so forth destroying his claim that mattes and transparencies were placed in the spacecraft windows to create the illusion of a faraway Earth.br br As with most films of this type Sibrel relies on innuendo inexpert assumption misleading commentary and selective quotation to manipulate the viewer into accepting a conclusion for which there is not a shred of actual evidence.\n"
     ]
    }
   ],
   "source": [
    "train_pos_sample_ind = np.random.randint(len(train_pos_data[0]))\n",
    "train_neg_sample_ind = np.random.randint(len(train_neg_data[0]))\n",
    "\n",
    "print(\"Positive Sentiment example\")\n",
    "print(train_pos_data[0][train_pos_sample_ind])\n",
    "print(\"---------------------------\")\n",
    "print(\"Negative Sentiment example\")\n",
    "print(train_neg_data[0][train_neg_sample_ind])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50000/50000 [01:37<00:00, 514.48it/s]\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "word_counts = defaultdict(int)\n",
    "\n",
    "# Compute the frequency of each unique\n",
    "for text in tqdm(train_texts + val_texts + test_texts):\n",
    "    # Splits sentences \n",
    "    for word in word_tokenize(text):\n",
    "        word_counts[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique words 210325\n",
      "Most frequent word:  the occurs 572555 times\n",
      "['<PAD>', 'the', '.', 'a', 'and', 'of', 'to', 'is', 'in', 'I', 'that', 'it', 'br', 'this', 'was', 'The', 'as', 'with', 'movie', 'for', 'film', 'but', 'on', 'are', 'not', 'have', 'his', 'you', 'be', '!', 'one', 'at', 'by', 'he', 'an', 'all', 'who', 'from', 'like', 'its', 'they', 'so', 'or', 'about', 'her', 'just', 'has', '?', 'out', 'This', 'some', 'good', 'more', 'very', '...', 'what', 'up', 'would', 'It', 'can', 'when', 'time', 'if', 'which', 'really', 'only', 'their', 'see', 'were', 'had', 'even', 'story', 'there', 'no', 'my', 'me', 'she', 'than', 'much', 'been', 'get', 'into', 'will', 'other', 'him', 'bad', 'because', 'people', 'do', 'great', 'well', 'most', 'we', 'them', 'first', 'made', 'also', 'movies', 'make', 'how']\n"
     ]
    }
   ],
   "source": [
    "vocab = ['<PAD>'] + sorted(word_counts, key=lambda word: word_counts[word], reverse=True)\n",
    "word2id = {word: i for i, word in enumerate(vocab)}\n",
    "\n",
    "# Examine the most common words\n",
    "print(\"Number of unique words\", len(vocab))\n",
    "print(\"Most frequent word: \", vocab[1], \"occurs\", word_counts[vocab[1]], \"times\")\n",
    "print(vocab[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savez('glove_embeddings.npz', embeddings=embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "glove_embeddings = np.load('glove_embeddings.npz')['embeddings']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def map_texts(texts, word2id):\n",
    "    return [[word2id[word] for word in word_tokenize(text)] for text in tqdm(texts)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 22500/22500 [00:42<00:00, 524.65it/s]\n",
      "100%|██████████| 2500/2500 [00:04<00:00, 530.67it/s]\n",
      "100%|██████████| 25000/25000 [00:47<00:00, 531.54it/s]\n"
     ]
    }
   ],
   "source": [
    "train_map_text = map_texts(train_texts, word2id)\n",
    "val_map_text = map_texts(val_texts, word2id)\n",
    "test_map_text = map_texts(test_texts, word2id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = keras.preprocessing.sequence.pad_sequences(train_map_text)\n",
    "x_val = keras.preprocessing.sequence.pad_sequences(val_map_text)\n",
    "x_test = keras.preprocessing.sequence.pad_sequences(test_map_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras import layers\n",
    "from keras import models\n",
    "\n",
    "def get_LSTM_model(embedding_matrix):\n",
    "    inp = layers.Input(shape=(None,))\n",
    "    x = layers.Embedding(*(embedding_matrix.shape),\n",
    "                         weights=[embedding_matrix], \n",
    "                         trainable=False)(inp)\n",
    "    x = layers.Bidirectional(layers.LSTM(50, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)\n",
    "    x = layers.GlobalMaxPool1D()(x)\n",
    "    x = layers.Dropout(0.1)(x)\n",
    "    x = layers.Dense(50, activation=\"relu\")(x)\n",
    "    x = layers.Dropout(0.1)(x)\n",
    "    x = layers.Dense(1)(x)\n",
    "    model = models.Model(inputs=inp, outputs=x)\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                  optimizer='adam',\n",
    "                  metrics=['accuracy'])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = get_LSTM_model(glove_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_7 (InputLayer)         (None, None)              0         \n",
      "_________________________________________________________________\n",
      "embedding_6 (Embedding)      (None, None, 300)         63097500  \n",
      "_________________________________________________________________\n",
      "bidirectional_5 (Bidirection (None, None, 100)         140400    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_3 (Glob (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 50)                5050      \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1)                 51        \n",
      "=================================================================\n",
      "Total params: 63,243,001\n",
      "Trainable params: 145,501\n",
      "Non-trainable params: 63,097,500\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 22500 samples, validate on 2500 samples\n",
      "Epoch 1/20\n",
      "17152/22500 [=====================>........] - ETA: 7:54 - loss: 1.1205 - acc: 0.5248"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-107-822756b0a081>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m           \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m128\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m           \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m           shuffle=True)\n\u001b[0m",
      "\u001b[0;32m~/anaconda/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1040\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1042\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1043\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1044\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m~/anaconda/lib/python3.6/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2659\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2660\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2661\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2662\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2663\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2629\u001b[0m                                 \u001b[0msymbol_vals\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2630\u001b[0m                                 session)\n\u001b[0;32m-> 2631\u001b[0;31m         \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2632\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2633\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m   1449\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_created_with_new_api\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1450\u001b[0m           return tf_session.TF_SessionRunCallable(\n\u001b[0;32m-> 1451\u001b[0;31m               self._session._session, self._handle, args, status, None)\n\u001b[0m\u001b[1;32m   1452\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1453\u001b[0m           return tf_session.TF_DeprecatedSessionRunCallable(\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.fit(x_train, \n",
    "          train_labels,\n",
    "          validation_data=(x_val, val_labels),\n",
    "          batch_size=128,\n",
    "          epochs=20,\n",
    "          shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}