predict_ratings.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Yelp Review Prediction\n",
    "Objective: Construct and train a Neural Network that would be able to predict the number of star ratings from a Yelp review.    \n",
    "Dataset used: https://www.yelp.com/dataset/challenge  \n",
    "\n",
    "Steps:  \n",
    "1) Data Preprocessing  \n",
    "2) Deep Learning Preprocessing  \n",
    "3) Network Training  \n",
    "4) Network Testing  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow Version: 1.5.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "%matplotlib inline\n",
    "from langdetect import detect\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas_ml import ConfusionMatrix\n",
    "import seaborn\n",
    "\n",
    "import sklearn as sk\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "import json\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.layers.core import Dense\n",
    "from tensorflow.python.ops.rnn_cell_impl import _zero_state_tensors\n",
    "import pickle\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "import nltk, re, time\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "from collections import defaultdict\n",
    "from collections import namedtuple\n",
    "from contractions import get_contractions\n",
    "\n",
    "import operator\n",
    "from tensorflow.python.ops import array_ops\n",
    "from tensorflow.python.ops import tensor_array_ops\n",
    "print('TensorFlow Version: {}'.format(tf.__version__))\n",
    "import os \n",
    "alreadyPickled = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\thoma\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variable Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_dim = 300\n",
    "num_layers = 2\n",
    "num_classes = 6\n",
    "epochs = 5\n",
    "batch_size = 64\n",
    "rnn_size = 64\n",
    "num_layers = 2\n",
    "\n",
    "learning_rate = 0.005\n",
    "\n",
    "keep_probability = 0.8\n",
    "max_sequence_length = 750\n",
    "\n",
    "IS_TRAINING = True\n",
    "IS_TESTING = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Loading\n",
    "If the data is already pickled, then can skip embedding and data processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def picklefiles(filename, stuff):\n",
    "    save_stuff = open(filename, \"wb\")\n",
    "    pickle.dump(stuff, save_stuff)\n",
    "    save_stuff.close()\n",
    "def loadfiles(filename):\n",
    "    saved_stuff = open(filename,\"rb\")\n",
    "    stuff = pickle.load(saved_stuff)\n",
    "    saved_stuff.close()\n",
    "    return stuff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing\n",
    "The following few cells preprocess the data for me. Clean test defines a function where it removes stop_words (found here https://gist.github.com/sebleier/554280). These words typically have no beneficial meaning to any reviews and are thus wasted features. I also strip punctuation and turn everything lower case. These procedures can be considered pretty standard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "    text = text.lower()    \n",
    "    if True:\n",
    "        text = text.split()\n",
    "        new_text = []\n",
    "        for word in text:\n",
    "            if word in contractions:\n",
    "                new_text.append(contractions[word])\n",
    "            else:\n",
    "                new_text.append(word)\n",
    "        text = \" \".join(new_text)\n",
    "    # Format words and remove unwanted characters\n",
    "    text = re.sub(r'https?:\\/\\/.*[\\r\\n]*', '', text, flags=re.MULTILINE)\n",
    "    text = re.sub(r'\\<a href', ' ', text)\n",
    "    text = re.sub(r'&amp;', '', text) \n",
    "    text = re.sub(r'[_\"\\-;%()|+&=*%.,!?:#$@\\[\\]/]', ' ', text)\n",
    "    text = re.sub(r'<br />', ' ', text)\n",
    "    text = re.sub(r'\\'', ' ', text)\n",
    "    text = text.split()\n",
    "    stops = set(stopwords.words(\"english\"))\n",
    "    text = [w for w in text if not w in stops]\n",
    "    text = \" \".join(text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 69047 restaurants\n"
     ]
    }
   ],
   "source": [
    "restId = []\n",
    "for line in open('./data/dataset/business.json', 'r'):\n",
    "    data = json.loads(line)\n",
    "    if 'Restaurants' in data['categories'] or 'Food' in data['categories']:\n",
    "        restId.append(data['business_id'])\n",
    "print(\"There are %d restaurants\" % (len(restId)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Grabbing the reviews and star ratings\n",
    "Here, I narrow down my reviews to english only reviews. I do this since restaurants make up around 60% of the yelp reviews. This way review types and wording may stay relatively similar for more accurate training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contractions = get_contractions()\n",
    "\n",
    "revs_list = [[]]\n",
    "stars_list = [[]]\n",
    "num = 1000000 # Number of review read\n",
    "k = 0 # Count\n",
    "nolang = [[]]\n",
    "for line in open('./data/dataset/review.json', 'r', encoding='utf-8'):\n",
    "    if k >= num:\n",
    "        break\n",
    "    data = json.loads(line)\n",
    "    text = data['text']\n",
    "    star = data['stars']\n",
    "    ID = data['business_id']\n",
    "    # Check language\n",
    "    if text == None:\n",
    "        continue\n",
    "    if star == None:\n",
    "        continue\n",
    "    if ID not in restId:\n",
    "        continue\n",
    "    try:\n",
    "        if detect(text) == 'en':\n",
    "            revs_list.append(clean_text(text))\n",
    "            stars_list.append(star)\n",
    "            k += 1\n",
    "            # Notify for every 5000 reviews\n",
    "            if len(revs_list) % 5000 == 0:\n",
    "                print(len(revs_list), k)\n",
    "    except:\n",
    "        nolang.append(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "love staff love meat love place prepare long line around lunch dinner hours ask want meat lean something maybe cannot remember say want fatty get half sour pickle hot pepper hand cut french fries\n",
      "1000001 1000001\n"
     ]
    }
   ],
   "source": [
    "print(revs_list[1])\n",
    "print(len(revs_list), len(stars_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1000001, 1) (1000001, 1) (1000001, 2)\n",
      "(1000001, 2)\n",
      "                                                text stars\n",
      "0                                                 []    []\n",
      "1  love staff love meat love place prepare long l...     5\n",
      "2  super simple place amazing nonetheless around ...     5\n",
      "3  small unassuming place changes menu every ofte...     5\n",
      "4  lester located beautiful neighborhood since 19...     5\n"
     ]
    }
   ],
   "source": [
    "np_revs = np.asarray([revs_list]).T\n",
    "np_stars = np.asarray([stars_list]).T\n",
    "stacked_revs = np.hstack((np_revs, np_stars))\n",
    "categories = ['text', 'stars']\n",
    "print(np_revs.shape, np_stars.shape, stacked_revs.shape)\n",
    "df_reviews_processing = pd.DataFrame(stacked_revs, columns=categories)\n",
    "print(df_reviews_processing.shape)\n",
    "print(df_reviews_processing.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before Dropping Nones: Shape 1000001,2\n",
      "After Dropping Nones: Shape 1000000,2\n"
     ]
    }
   ],
   "source": [
    "df_reviews_processing[['stars']] = df_reviews_processing[['stars']].apply(pd.to_numeric)\n",
    "\n",
    "# Grabbing only numbers that are of numerical value (get rid of None, NaN, etc)\n",
    "print(\"Before Dropping Nones: Shape %d,%d\" % (df_reviews_processing.shape[0], df_reviews_processing.shape[1]))\n",
    "\n",
    "df_reviews_processing = df_reviews_processing[np.isfinite(df_reviews_processing['stars'])]\n",
    "df_reviews = df_reviews_processing[np.isfinite(df_reviews_processing['stars'])]\n",
    "df_reviews = df_reviews.dropna()\n",
    "df_reviews = df_reviews.reset_index(drop=True)\n",
    "\n",
    "print(\"After Dropping Nones: Shape %d,%d\" % (df_reviews.shape[0], df_reviews.shape[1]))\n",
    "\n",
    "df_reviews.to_csv(\"./csvs/reviews_df_processed.csv\", encoding='utf-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Balance the Dataset\n",
    "Want to balance the dataset, such that we have an equal number of reviews for each different category.  \n",
    "For example, if our distribution of reviews is [200,500,100,300,400], for [1,2,3,4,5] stars, respectively, then I will only take 100 of each review  \n",
    "I do this so we have an equal representation of all labels when he train and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def balance_dataframe(df, category=['stars']):\n",
    "    \"\"\"\n",
    "    :param df: pandas.DataFrame\n",
    "    :param categorical_columns: iterable of categorical columns names contained in {df}\n",
    "    :return: balanced pandas.DataFrame\n",
    "    \"\"\"    \n",
    "    if category is None or not all([col in df.columns for col in category]):\n",
    "        raise ValueError('Please provide one or more columns containing categorical variables')\n",
    "\n",
    "    lowest_count = df.groupby(category).apply(lambda x: x.shape[0]).min()\n",
    "    df = df.groupby(category).apply( \n",
    "        lambda x: x.sample(lowest_count)).drop(category, axis=1).reset_index().set_index('level_1')\n",
    "\n",
    "    df.sort_index(inplace=True)\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(pre) 1 star ratings: 114331\n",
      "(pre) 2 star ratings: 94828\n",
      "(pre) 3 star ratings: 134510\n",
      "(pre) 4 star ratings: 269225\n",
      "(pre) 5 star ratings: 387071\n",
      "(post) 1 star ratings: 94828\n",
      "(post) 2 star ratings: 94828\n",
      "(post) 3 star ratings: 94828\n",
      "(post) 4 star ratings: 94828\n",
      "(post) 5 star ratings: 94828\n"
     ]
    }
   ],
   "source": [
    "df_reviews = pd.read_csv(\"./csvs/reviews_df_processed.csv\")\n",
    "df_reviews['len'] = df_reviews.text.str.len()\n",
    "\n",
    "df_reviews = df_reviews[df_reviews['len'].between(10, 4000)]\n",
    "df_reviews[['stars']] = df_reviews[['stars']].apply(pd.to_numeric)\n",
    "\n",
    "print(\"(pre) 1 star ratings: %d\" % (len(df_reviews[df_reviews.stars == 1])))\n",
    "print(\"(pre) 2 star ratings: %d\" % (len(df_reviews[df_reviews.stars == 2])))\n",
    "print(\"(pre) 3 star ratings: %d\" % (len(df_reviews[df_reviews.stars == 3])))\n",
    "print(\"(pre) 4 star ratings: %d\" % (len(df_reviews[df_reviews.stars == 4])))\n",
    "print(\"(pre) 5 star ratings: %d\" % (len(df_reviews[df_reviews.stars == 5])))\n",
    "\n",
    "df_balanced = balance_dataframe(df_reviews, \n",
    "                                category=['stars'])\n",
    "\n",
    "df_balanced.to_csv('balanced_reviews1000.csv', encoding='utf-8')\n",
    "print(\"(post) 1 star ratings: %d\" % (len(df_balanced[df_balanced.stars == 1])))\n",
    "print(\"(post) 2 star ratings: %d\" % (len(df_balanced[df_balanced.stars == 2])))\n",
    "print(\"(post) 3 star ratings: %d\" % (len(df_balanced[df_balanced.stars == 3])))\n",
    "print(\"(post) 4 star ratings: %d\" % (len(df_balanced[df_balanced.stars == 4])))\n",
    "print(\"(post) 5 star ratings: %d\" % (len(df_balanced[df_balanced.stars == 5])))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         stars  Unnamed: 0                                               text  \\\n",
      "level_1                                                                         \n",
      "4          4.0           4  love coming yes place always needs floor swept...   \n",
      "6          4.0           6  would guess would able get fairly decent vietn...   \n",
      "8          3.0           8  bad love gluten free vegan version cheese curd...   \n",
      "9          4.0           9  currently parents new favourite restaurant com...   \n",
      "10         3.0          10  server little rude ordered calamari duck confi...   \n",
      "\n",
      "           len  \n",
      "level_1         \n",
      "4        314.0  \n",
      "6        376.0  \n",
      "8        153.0  \n",
      "9        266.0  \n",
      "10       107.0  \n",
      "566.0\n",
      "660.0\n",
      "721.0\n",
      "1045.0\n"
     ]
    }
   ],
   "source": [
    "print(df_balanced.head())\n",
    "print(np.percentile(df_balanced.len, 80))\n",
    "print(np.percentile(df_balanced.len, 85))\n",
    "print(np.percentile(df_balanced.len, 87.5))\n",
    "print(np.percentile(df_balanced.len, 95))\n",
    "max_sequence_length = 750"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Embeddings\n",
    "Using ConceptNet Numberbatch instead of GLoVE (supposedly outperforms GLoVE embeddings)  \n",
    "https://github.com/commonsense/conceptnet-numberbatch\n",
    "  \n",
    "On top of the embeddings, we also keep track of commonly used words in the reviews that Embeddings don't cover. This way we could have higher test accuracy when words we come across words like these. This is specified by a threshold value. Currently, threshold is set to 20 occuraces.  \n",
    "  \n",
    "  \n",
    "We also process the reviews a bit more, sorting them into comparable lengths. This way, there is less padding necessary and (possibly) faster computation time when training/testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_words(count_dict, text):\n",
    "    '''Count the number of occurrences of each word in a set of text'''\n",
    "    for sentence in text:\n",
    "        for word in sentence.split():\n",
    "            if word not in count_dict:\n",
    "                count_dict[word] = 1\n",
    "            else:\n",
    "                count_dict[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of Vocabulary: 143090\n"
     ]
    }
   ],
   "source": [
    "word_counts = {}\n",
    "count_words(word_counts, df_balanced.text)            \n",
    "print(\"Size of Vocabulary:\", len(word_counts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_path='./embeddings/numberbatch-en.txt'\n",
    "def load_embeddings(path='./embeddings/numberbatch-en.txt'):\n",
    "    embeddings_index = {}\n",
    "    with open(path, encoding='utf-8') as f:\n",
    "        for line in f:\n",
    "            values = line.split(' ')\n",
    "            word = values[0]\n",
    "            embedding = np.asarray(values[1:], dtype='float32')\n",
    "            embeddings_index[word] = embedding\n",
    "    return embeddings_index\n",
    "\n",
    "embeddings_index = load_embeddings(embed_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenizer\n",
    "Here we will create a dictionary __word2int__ that will map each word to respective value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of unique words: 143090\n",
      "Number of words we will use: 66738\n",
      "Percent of words we will use: 46.64%\n"
     ]
    }
   ],
   "source": [
    "#dictionary to tokenizer words\n",
    "word2int = {} \n",
    "threshold = 20\n",
    "token_index = 0\n",
    "for word, count in word_counts.items():\n",
    "    if count >= threshold or word in embeddings_index:\n",
    "        word2int[word] = token_index\n",
    "        token_index += 1\n",
    "\n",
    "# Special tokens that will be added to our vocab\n",
    "special_characters = [\"<unk>\",\"<pad>\"]   \n",
    "\n",
    "# Add codes to vocab\n",
    "for c in special_characters:\n",
    "    word2int[c] = len(word2int)\n",
    "    \n",
    "usage_ratio = round(len(word2int) / len(word_counts),4)*100\n",
    "\n",
    "print(\"Total number of unique words:\", len(word_counts))\n",
    "print(\"Number of words we will use:\", len(word2int))\n",
    "print(\"Percent of words we will use: {}%\".format(usage_ratio))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66738 66738\n"
     ]
    }
   ],
   "source": [
    "# Need to use 300 for embedding dimensions to match CN's vectors.\n",
    "embedding_dim = 300\n",
    "nb_words = len(word2int)\n",
    "\n",
    "# Create matrix with default values of zero\n",
    "word_embedding_matrix = np.zeros((nb_words, embedding_dim), dtype=np.float32)\n",
    "for word, i in word2int.items():\n",
    "    if word in embeddings_index:\n",
    "        word_embedding_matrix[i] = embeddings_index[word]\n",
    "    else:\n",
    "        # If word not in embeddings, create a random embedding for it\n",
    "        new_embedding = np.array(np.random.uniform(-1.0, 1.0, embedding_dim))\n",
    "        embeddings_index[word] = new_embedding\n",
    "        word_embedding_matrix[i] = new_embedding\n",
    "\n",
    "# Check if value matches len(word2int). \n",
    "print(len(word_embedding_matrix), len(word2int))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenize Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_to_ints(text, pred=False):\n",
    "    if pred:\n",
    "        seq = []\n",
    "        for word in text.split():\n",
    "            if word in word2int:\n",
    "                seq.append(word2int[word])\n",
    "            else:\n",
    "                seq.append(word2int[\"<unk>\"])\n",
    "        return seq\n",
    "    else:\n",
    "        seq = []\n",
    "        for s in text:\n",
    "            temp_seq = []\n",
    "            for word in s.split():\n",
    "                if word in word2int:\n",
    "                    temp_seq.append(word2int[word])\n",
    "                else:\n",
    "                    temp_seq.append(word2int[\"<unk>\"])\n",
    "            seq.append(temp_seq)\n",
    "        return seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq = convert_to_ints(df_balanced['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the ratings into One-Hot representation\n",
    "ratings = df_balanced.stars.values.astype(int)\n",
    "ratings_cat = tf.keras.utils.to_categorical(ratings)\n",
    "X_train, X_test, y_train, y_test = train_test_split(seq, ratings_cat, test_size=0.2, random_state=9)\n",
    "with pd.HDFStore('x_y_test_train.h5') as h:\n",
    "    h['X_train'] = pd.DataFrame(X_train)\n",
    "    h['X_test'] = pd.DataFrame(X_test)\n",
    "    h['y_train'] = pd.DataFrame(y_train)\n",
    "    h['y_test'] = pd.DataFrame(y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting Batches\n",
    "Gets batches. These will be called later to then fill our X and y placeholders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_batch(batch):\n",
    "    # Want to pad this way since tensorflow preprocessing pads with 0's, which can eventually lead to zero tensors\n",
    "    lengths = []\n",
    "    for text in batch:\n",
    "        lengths.append(len(text))\n",
    "    max_length = max(lengths)\n",
    "    pad_text = tf.keras.preprocessing.sequence.pad_sequences(batch, \n",
    "                                                             maxlen=max_length, \n",
    "                                                             padding='post', \n",
    "                                                             value=word2int['<pad>'])\n",
    "    return pad_text\n",
    "\n",
    "def get_batches(x, y, batch_size):\n",
    "    # Make sure to not exceed amount of data\n",
    "    for batch_i in range(0, len(x)//batch_size):\n",
    "        start = batch_i * batch_size\n",
    "        end = start+batch_size\n",
    "        batch_x = x[start:end]\n",
    "        labels = y[start:end]\n",
    "        pad_batch_x = np.array(pad_batch(batch_x))\n",
    "        yield pad_batch_x, labels\n",
    "        \n",
    "def get_test_batches(x, batch_size):\n",
    "    for batch_i in range(0, len(x)//batch_size):\n",
    "        start = batch_i * batch_size\n",
    "        end = start+batch_size\n",
    "        batch = x[start:end]\n",
    "        pad_batch_test = np.array(pad_batch(batch))\n",
    "        yield pad_batch_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "if alreadyPickled == False:\n",
    "    \n",
    "    picklefiles(\"./data/pickles/balanced_reviews.p\",df_balanced)\n",
    "    picklefiles(\"./data/pickles/category_ratings.p\",ratings_cat)\n",
    "    picklefiles(\"./data/pickles/word_embedding_matrix.p\",word_embedding_matrix)\n",
    "    picklefiles(\"./data/pickles/word2int.p\", word2int)\n",
    "    \n",
    "if alreadyPickled == True:\n",
    "    \n",
    "    word_embedding_matrix = loadfiles(\"./data/pickles/word_embedding_matrix.p\")\n",
    "    ratings_cat = loadfiles(\"./data/pickles/category_ratings.p\")\n",
    "    df_balanced = pd.read_csv('balanced_reviews.csv')\n",
    "    balanced_reviews = loadfiles(\"./data/pickles/balanced_reviews.p\")\n",
    "    word2int = loadfiles('./data/pickles/word2int.p')\n",
    "    with pd.HDFStore('x_y_test_train.h5') as h:\n",
    "        X_train = h['X_train'].values\n",
    "        X_test = h['X_test'].values\n",
    "        y_train = h['y_train'].values\n",
    "        y_test = h['y_test'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Building the Graph\n",
    "Here we start building our computational graph. We use a 2 layer GRU Recurrent Neural Network. We define placeholders for learning rate and dropout since these are variables that we could potentially want to vary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_inputs():\n",
    "    # Should be [batch_size x review length]\n",
    "    input_data = tf.placeholder(tf.int32, [None, None], name='input')\n",
    "    # Should be [batch_size x num_classes]\n",
    "    labels = tf.placeholder(tf.int32, [None, None], name='labels')\n",
    "    lr = tf.placeholder(tf.float32, name='learning_rate')\n",
    "    keep_prob = tf.placeholder(tf.float32, name='keep_prob')\n",
    "    return input_data, labels, lr, keep_prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reset the graph\n",
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Graph is built.\n",
      "./graph\n"
     ]
    }
   ],
   "source": [
    "train_graph = tf.Graph()\n",
    "with train_graph.as_default():\n",
    "    with tf.name_scope(\"inputs\"):\n",
    "        input_data, labels, lr, keep_prob = model_inputs()\n",
    "        weight = tf.Variable(tf.truncated_normal([rnn_size, num_classes], stddev=(1/np.sqrt(rnn_size*num_classes))))\n",
    "        bias = tf.Variable(tf.constant(0.1, shape=[num_classes]))\n",
    "\n",
    "    embeddings = word_embedding_matrix\n",
    "    embs = tf.nn.embedding_lookup(embeddings, input_data)\n",
    "\n",
    "    with tf.name_scope(\"RNN_Layers\"):\n",
    "        \n",
    "        stacked_rnn = []\n",
    "        for layer in range(num_layers):\n",
    "            cell_fw = tf.contrib.rnn.GRUCell(rnn_size)\n",
    "            cell_fw = tf.contrib.rnn.DropoutWrapper(cell_fw,\n",
    "                                                    output_keep_prob=keep_prob)\n",
    "            stacked_rnn.append(cell_fw)\n",
    "        multilayer_cell = tf.contrib.rnn.MultiRNNCell(stacked_rnn, state_is_tuple=True)\n",
    "\n",
    "        \n",
    "    with tf.name_scope(\"init_state\"):\n",
    "        initial_state = multilayer_cell.zero_state(batch_size, tf.float32)\n",
    "\n",
    "    with tf.name_scope(\"Forward_Pass\"):\n",
    "        output, final_state = tf.nn.dynamic_rnn(multilayer_cell,\n",
    "                                           embs,\n",
    "                                           dtype=tf.float32)\n",
    "\n",
    "    with tf.name_scope(\"Predictions\"):\n",
    "        last = output[:, -1, :]\n",
    "        predictions = tf.exp(tf.matmul(last, weight) + bias)\n",
    "        tf.summary.histogram('predictions', predictions)\n",
    "        \n",
    "    with tf.name_scope('cost'):\n",
    "        cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=predictions, labels=labels))        \n",
    "        tf.summary.scalar('cost', cost)\n",
    "    \n",
    "    # Optimizer \n",
    "    with tf.name_scope('train'):    \n",
    "        optimizer = tf.train.AdamOptimizer(lr).minimize(cost)\n",
    "    \n",
    "    # Predictions comes out as 6 output layer, so need to \"change\" to one hot\n",
    "    with tf.name_scope(\"accuracy\"):\n",
    "        correctPred = tf.equal(tf.argmax(predictions,1), tf.argmax(labels,1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correctPred, tf.float32))\n",
    "        tf.summary.scalar('accuracy', accuracy)\n",
    "    \n",
    "    export_nodes = ['input_data', 'labels', 'keep_prob', 'lr', 'initial_state', 'final_state',\n",
    "                    'accuracy', 'predictions', 'cost', 'optimizer', 'merged']\n",
    "    \n",
    "    merged = tf.summary.merge_all()\n",
    "    \n",
    "print(\"Graph is built.\")\n",
    "graph_location = \"./graph\"\n",
    "\n",
    "Graph = namedtuple('train_graph', export_nodes)\n",
    "local_dict = locals()\n",
    "graph = Graph(*[local_dict[each] for each in export_nodes])\n",
    "    \n",
    "print(graph_location)\n",
    "train_writer = tf.summary.FileWriter(graph_location)\n",
    "train_writer.add_graph(train_graph)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate_decay = 0.95\n",
    "min_learning_rate = 0.0005\n",
    "display_step = 20 # Check training loss after every 20 batches\n",
    "stop_early = 0 \n",
    "stop = 3 # If the update loss does not decrease in 3 consecutive update checks, stop training\n",
    "per_epoch = 8 # Make 8 update checks per epoch\n",
    "update_check = (len(seq)//batch_size//per_epoch)-1\n",
    "keep_probability = 0.75\n",
    "update_loss = 0 \n",
    "batch_loss = 0\n",
    "summary_update_loss = [] # Record the update losses for saving improvements in the model\n",
    "shuffle = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shuffle_data(X, y):\n",
    "    whole_data = np.insert(X, 0, y, axis=1)\n",
    "    np.random.shuffle(whole_data)\n",
    "    labels = whole_data[0,:]\n",
    "    data = whole_data[1,:]\n",
    "    return data, labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training the Model\n",
    "I've been keeping track of the tensorboard summaries so it'll allow me to visualize the training process. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "load = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = \"./saves/best_model.ckpt\" \n",
    "if load:\n",
    "    loader = tf.train.import_meta_graph(\"./\" + checkpoint + '.meta')\n",
    "if IS_TRAINING:\n",
    "    with tf.Session(graph=train_graph) as sess:\n",
    "        if load:\n",
    "            loader.restore(sess, checkpoint)\n",
    "        else:\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "\n",
    "        train_writer = tf.summary.FileWriter('./summaries' + '/train', sess.graph)\n",
    "\n",
    "        for epoch_i in range(1, epochs+1):\n",
    "            state = sess.run(graph.initial_state)\n",
    "\n",
    "            update_loss = 0\n",
    "            batch_loss = 0\n",
    "\n",
    "            for batch_i, (x, y) in enumerate(get_batches(X_train, y_train, batch_size)):\n",
    "                if batch_i == 1:\n",
    "                    print(\"Starting\")\n",
    "                feed = {graph.input_data: x,\n",
    "                        graph.labels: y,\n",
    "                        graph.keep_prob: keep_probability,\n",
    "                        graph.initial_state: state, \n",
    "                        graph.lr: learning_rate}\n",
    "                start_time = time.time()\n",
    "                summary, loss, acc, state, _ = sess.run([graph.merged, \n",
    "                                                         graph.cost, \n",
    "                                                         graph.accuracy, \n",
    "                                                         graph.final_state, \n",
    "                                                         graph.optimizer], \n",
    "                                                        feed_dict=feed)\n",
    "                if batch_i == 1:\n",
    "                    print(\"Finished first\")\n",
    "\n",
    "                train_writer.add_summary(summary, epoch_i*batch_i + batch_i)\n",
    "\n",
    "                batch_loss += loss\n",
    "                update_loss += loss\n",
    "                end_time = time.time()\n",
    "                batch_time = end_time - start_time\n",
    "\n",
    "                if batch_i % display_step == 0 and batch_i > 0:\n",
    "                    print('Epoch {:>3}/{} Batch {:>4}/{} - Loss: {:>6.3f}, Acc: {:>6.3f}, Seconds: {:>4.2f}'\n",
    "                          .format(epoch_i,\n",
    "                                  epochs, \n",
    "                                  batch_i, \n",
    "                                  len(X_train) // batch_size, \n",
    "                                  batch_loss / display_step,\n",
    "                                  acc,\n",
    "                                  batch_time*display_step))\n",
    "                    batch_loss = 0\n",
    "\n",
    "                if batch_i % update_check == 0 and batch_i > 0:\n",
    "                    print(\"Average loss for this update:\", round(update_loss/update_check,3))\n",
    "                    summary_update_loss.append(update_loss)\n",
    "\n",
    "                    # If the update loss is at a new minimum, save the model\n",
    "                    if update_loss <= min(summary_update_loss):\n",
    "                        print('New Record!') \n",
    "                        stop_early = 0\n",
    "                        saver = tf.train.Saver() \n",
    "                        saver.save(sess, checkpoint)\n",
    "\n",
    "                    else:\n",
    "                        print(\"No Improvement.\")\n",
    "                        stop_early += 1\n",
    "                        if stop_early == stop:\n",
    "                            break\n",
    "                    update_loss = 0\n",
    "\n",
    "\n",
    "            # Reduce learning rate, but not below its minimum value\n",
    "            learning_rate *= learning_rate_decay\n",
    "            if learning_rate < min_learning_rate:\n",
    "                learning_rate = min_learning_rate\n",
    "            # Set shuffle to True if you want to shuffle data between epochs\n",
    "            # This can add some randomness and potentially learn new patterns in data\n",
    "#             if shuffle:\n",
    "#                 X_train, y_train = shuffle_data(X_train, y_train)\n",
    "            if stop_early == stop:\n",
    "                print(\"Stopping Training.\")\n",
    "                break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing the Data\n",
    "This part of the code is allocated to testing the data. On top of recording accuracy results, I also generate a confusion matrix. Since reviews are subjective and aren't concretely one rating or another, a confusion matrix helps visualize your results a lot better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "IS_TESTING = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if IS_TESTING:\n",
    "    with tf.Session(graph=train_graph) as sess:\n",
    "        checkpoint = \"./saves/best_model.ckpt\"  \n",
    "\n",
    "        all_preds = []\n",
    "\n",
    "        with tf.Session() as sess:\n",
    "            saver = tf.train.Saver()\n",
    "            # Load the model\n",
    "            saver.restore(sess, checkpoint)\n",
    "            test_state = sess.run(graph.initial_state)\n",
    "            print(\"Total Batches: %d\"%(len(X_test)//batch_size))\n",
    "            for ii, x in enumerate(get_test_batches(X_test, batch_size), 1):\n",
    "                if ii%20==0:\n",
    "                    print(\"%d batches\"%(ii))\n",
    "                feed = {graph.input_data: x,\n",
    "                        graph.keep_prob: keep_probability,\n",
    "                        graph.initial_state: state}\n",
    "\n",
    "                predictions = sess.run(graph.predictions, feed_dict=feed)\n",
    "                for i in range(len(predictions)):\n",
    "                    all_preds.append(predictions[i,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(94784, 6)\n",
      "0.619281735313977\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEWCAYAAAB7QRxFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAHlpJREFUeJzt3XmcXHWd7vHPk7CEnSCCmoCABgT0GiQy94oiKGBQBGcUBTdwlFzugBuig6MXEUdhVNS5M9EhKi6gouKWwYzoVYKySQKGJQEkbNIEZRUEwtLdz/xxTmOl0lVd3anqqlM8b17nRddZv6e6861v/c7v/I5sExER1TWl2wFERMS6SSKPiKi4JPKIiIpLIo+IqLgk8oiIiksij4iouCTyLpO0kaT/lPSApO+vw37eIunn7YytGyT9l6Qjux1HL5O0gyRLWq98PaH3TNL2kh6SNLX9UcZkSiJvkaQ3S1pa/uHfWf7jeWkbdv0GYFvgabYPm+hObH/L9oFtiGcNkvYtk8YP6+a/sJy/uMX9nCzp7LHWs32Q7W9MIM4NJJ0uaaD8Hd0i6fM1y2+VtP9491sFrb5n9e+B7T/Y3tT2UGcjjE5LIm+BpOOBLwCfoki62wNfBA5tw+6fDfze9mAb9tUpdwMvkfS0mnlHAr9v1wFUWJe/xw8Dc4C9gM2A/YDftSM2gE5VrW047wiwnanJBGwBPAQc1mSdDSkS/apy+gKwYblsX2AA+ABwF3An8I5y2ceBx4EnymO8EzgZOLtm3zsABtYrXx8F3Az8BbgFeEvN/ItqtnsJsAR4oPz/S2qWLQY+AVxc7ufnwNYNzm0k/v8Aji3nTS3nnQQsrln3X4HbgQeBK4CXlfPn1p3nVTVxfLKMYzXw3HLeu8rlXwLOrdn/vwC/BDRKnOcB72twDmcBw+UxHgI+VM7/PvDH8j36NbB7zTZfL4+/CHgY2B94NbCifM/uAE5ocLyjynP6t3Lf1wOvrHv/6897C+Cr5d/HHcA/A1Nr3u/PAveUv/tj6/4mnnzPytdHA9eVca4AXjTae8Daf1vPAhYC9wErgaNr9nky8D3gm+V+lwNzuv3vM1P5++l2AL0+lUlocOSPvcE6pwCXAdsATwcuAT5RLtu33P4UYP0yGTwCTC+Xn8yaibv+9ZP/2IBNKJLkLuWyZ44kH2oSObAVcD/wtnK7I8rXTyuXLwZuAnYGNipfn9bg3PalSNovAX5bzns1cD7wLtZM5G8FnlYe8wMUSXLaaOdVE8cfgN3LbdZnzUS+MUXVfxTwMopENrNBnB8t9/UPwAuoS/bArcD+dfP+nqJ6H/kgXlaz7OsUSXhvim+u0yiS7MiH03TgRQ1iOar8nb+/PKc3lfvaqsl5/xg4o/wdbwNcDvzvcv1jKD4Mtit/txfQIJEDh1F8ELwYEMWHxLNHew9YO5FfSPFNcxowm+Kb2Ctrfn+Plr/7qcCpwGXd/veZqZjylW5sTwPucfOmj7cAp9i+y/bdFJX222qWP1Euf8L2IoqKaJcJxjMMPF/SRrbvtL18lHVeA9xo+yzbg7a/Q5EIXluzztds/972aopKa3azg9q+BNhK0i7A2ykqs/p1zrZ9b3nM0ykS5Fjn+XXby8ttnqjb3yMUHw6fA84G3m17oMF+TqWo2N8CLAXuGOsCoO0zbf/F9mMUieqFkraoWeUnti+2PWz7UYrf426SNrd9v+0rm+z+LuAL5e/8u8ANFL+Xtc6bIjkfRPGN4mHbdwGfBw4v131jua/bbd9Xnmsj7wI+bXuJCytt39bsfQCQtB3wUuAfbT9qexnwFdb8O77I9iIXbepnAS8ca78xOZLIx3YvsPVID4EGngXU/mO5rZz35D7qPggeATYdbyC2H6ao7o4B7pT0U0nPayGekZhm1Lz+4wTiOQs4jqL9+Uf1CyV9QNJ1ZQ+cP1M0F2w9xj5vb7bQ9uUUzQmi+MBptN6Q7fm29wa2pGi6OFPSrqOtL2mqpNMk3STpQYpqlbp462N7PUVFepukCyX9ryah32G7dkS6+r+J2n0/m6Iqv1PSn8v37gyKypxyu9r1myXm7Si+bY3Xs4D7bP+l7jjN/mamjfHvIiZJEvnYLqX4Svm6JuusovjHOGL7ct5EPEzRpDDiGbULbZ9v+wCKZpXrgS+3EM9ITHdMMKYRZ1E0XSwqq+UnSXoZ8I8U1eN021tSNCdoJPQG+2w6/KakYykq+1UU7bpjsr3a9nyK5qTdGhznzRQXq/en+MDZYeSQjWIrq9xDKRLsj2nywQLMkFS7r/q/idp93w48RnGdYsty2tz27uXyOykSdO2+GrkdeE6DZc3e61UU37g2qzvOuv7NxCRIIh+D7QcoLurNl/Q6SRtLWl/SQZI+Xa72HeCjkp4uaety/TG72jWwDNin7OO7BUVvDAAkbSvpEEmbUPzDfwgYrevYImDnssvkepLeRJHQzptgTADYvgV4OfCRURZvRtEufDewnqSTgM1rlv8J2GE8PTQk7Uxx0e+tFF/xPyRp1CYgSe8ru0puVJ7zkWVMIz1X/gTsVBfvYxTfuDam6JHULJYNyr76W5RNQA8y+ns/YhvgPeXfymHArhS/l7XYvpPigvPpkjaXNEXScyS9vFzle+W+ZkqaDpzY5LhfAU6QtGfZI+a5kkY+1Ovfg9oYbqe4tnOqpGmS/gfFxfdvNTlW9Igk8hbY/hxwPMUFtbspqp7jKKoyKJLNUuBq4BrgynLeRI71C+C75b6uYM3kO4XiIuIqip4FL6eokOv3cS9wcLnuvRSV7MG275lITHX7vsj2aN82zgf+i+Li5G0U32JqmwNGbna6V1KztmUAyq/sZwP/Yvsq2zcC/wScJWnDUTZZDZxO8fX/HoqeHa+3fXO5/FSKD9s/SzqBoo3/NoqKcwXFxeqxvA24tWyKOYbiA6aR3wKzylg+Cbyh/L008nZggzKW+4FzKb51QfGt63zgKoq/rR+OtgMA298vj/dtit4lP6Zog4e134N6R1B8M1lF0XT2sfLvMXqc1mzGi4h1Jekoil4k7bhhLGJMqcgjIiouiTwiouLStBIRUXGpyCMiKq5nO/M/PnBN331VeMveH+12CB1x3l3Luh1C2z0x1MtjmE2cxl6lkp54/I51PrUn7rm55Zyz/tY79dRbmYo8IqLierYij4iYVMPVHZY9iTwiAqDCzWlJ5BERgD3c7RAmLIk8IgJgOIk8IqLaUpFHRFRcLnZGRFRcKvKIiGpzhXut5IagiAgoLna2Oo1B0lxJN0haKWmtB4FI+rykZeX0+/LxfiPLhmqWLWwl9FTkERHQtqYVSVOB+cABwACwRNJC2yuePJT9/pr13w3sUbOL1babPgy9XiryiAgoLna2OjW3F7DS9s22HwfOoXg+bCNHUDwucsKSyCMioKjIW5wkzZO0tGaaV7OnGaz5mMOBct5ayuep7gj8qmb2tHKfl0lq9tD3J6VpJSICxnWLvu0FwIIGi0cbGbHRyIqHA+fari3zt7e9StJOwK8kXWP7pmbxpCKPiIB2XuwcALareT2T4oHWozmcumaVkYeblw8OX8ya7eejSiKPiADsoZanMSwBZknaUdIGFMl6rd4nknYBpgOX1sybLmnD8uetgb2BFfXb1kvTSkQEtK3Xiu1BSccB5wNTgTNtL5d0CrDU9khSPwI4x2s+b3NX4AxJwxSF9mm1vV0aSSKPiIC2DpplexGwqG7eSXWvTx5lu0uAF4z3eEnkERGQW/QjIipv6IluRzBhk36xU9I7JvuYERFjauMt+pOtG71WPt5oQW0n+69869zJjCkinurGcUNQr+lI04qkqxstArZttF1tJ/vHB65p1IE+IqL9erDSblWn2si3BV4F3F83X8AlHTpmRMTEJZGv5TxgU9vL6hdIWtyhY0ZETJgrfLGzI4nc9jubLHtzJ44ZEbFOerDtu1XpfhgRAWlaiYiovFTkEREVl4o8IqLiUpFHRFTcYOsPlug1SeQREZCKPCKi8tJGHhFRcanIIyIqLhV5RETFpSKPiKi49FqJiKg4V3fk7CTyiAhIG3lEROUlkUdEVFwudkZEVNzQULcjmLCeTeT/96Wf6nYIbfe1w7odQWf83Xd37XYIbXfxvdd3O4SOGByubrLquDStRERUXBJ5RETFpY08IqLaPJx+5BER1ZamlYiIikuvlYiIiktFHhFRcRVO5FO6HUBERE+wW5/GIGmupBskrZR0YoN13ihphaTlkr5dM/9ISTeW05GthJ6KPCIC2laRS5oKzAcOAAaAJZIW2l5Rs84s4MPA3rbvl7RNOX8r4GPAHMDAFeW29zc7ZiryiAiAYbc+NbcXsNL2zbYfB84BDq1b52hg/kiCtn1XOf9VwC9s31cu+wUwd6wDJpFHREDRa6XFSdI8SUtrpnk1e5oB3F7zeqCcV2tnYGdJF0u6TNLccWy7ljStREQAHkfTiu0FwIIGizXaJnWv1wNmAfsCM4HfSHp+i9uuJRV5RAS0s2llANiu5vVMYNUo6/zE9hO2bwFuoEjsrWy7liTyiAgoxlppdWpuCTBL0o6SNgAOBxbWrfNjYD8ASVtTNLXcDJwPHChpuqTpwIHlvKbStBIRAa1U2i2xPSjpOIoEPBU40/ZySacAS20v5K8JewUwBHzQ9r0Akj5B8WEAcIrt+8Y6ZhJ5RATAYPtu0be9CFhUN++kmp8NHF9O9dueCZw5nuMlkUdEQIaxjYiovAxjGxFRbePpfthrksgjIqDSFXnHuh9Kep6kV0ratG7+mLebRkRMuvb1I590HUnkkt4D/AR4N3CtpNpxBj7ViWNGRKyTcdyi32s61bRyNLCn7Yck7QCcK2kH2//K6LegAlCOVzAP4MCt5jB7s+d2KLyIiDVV+ZmdnWpamWr7IQDbt1KMJ3CQpM/RJJHbXmB7ju05SeIRManStLKWP0qaPfKiTOoHA1sDL+jQMSMiJm54uPWpx3SqaeXtwGDtDNuDwNslndGhY0ZETFwPVtqt6kgitz3QZNnFnThmRMQ6SSKPiKg2D/Vek0mrksgjIiAVeURE1VW5+2ESeUQEpCKPiKi86jaRJ5FHRAB4sLqZPIk8IgJSkUdEVF0udkZEVF0q8oiIaktFHhFRdanIIyKqzYNjr9OrksgjIgCnIo+IqLgk8oiIaktFHhFRcUnkHfDV+67odght53P37HYIHfHNHR/odghtd8KUF3U7hI742f3XdjuEnuWhho8T7nk9m8gjIiZTKvKIiIrzcCryiIhKS0UeEVFxdnUr8indDiAiohd4uPVpLJLmSrpB0kpJJzZZ7w2SLGlO+XoHSaslLSun/2gl9lTkERHAcJt6rUiaCswHDgAGgCWSFtpeUbfeZsB7gN/W7eIm27PHc8xU5BERFBc7W53GsBew0vbNth8HzgEOHWW9TwCfBh5d19iTyCMiGF8ilzRP0tKaaV7NrmYAt9e8HijnPUnSHsB2ts8bJZQdJf1O0oWSXtZK7GlaiYgAPI7hyG0vABY0WDxayf7k3iVNAT4PHDXKencC29u+V9KewI8l7W77wWbxNEzkkv6z9uBrRWUf0mzHERFV0sZ+5APAdjWvZwKral5vBjwfWCwJ4BnAQkmH2F4KPAZg+wpJNwE7A0ubHbBZRf7ZcYcfEVFRbex+uASYJWlH4A7gcODNfz2OHwC2HnktaTFwgu2lkp4O3Gd7SNJOwCzg5rEO2DCR275womcREVE1Q23qtWJ7UNJxwPnAVOBM28slnQIstb2wyeb7AKdIGgSGgGNs3zfWMcdsI5c0CzgV2A2YVhPsTmNtGxFRFe28Icj2ImBR3byTGqy7b83PPwB+MN7jtdJr5WvAl4BBYD/gm8BZ4z1QREQva2P3w0nXSiLfyPYvAdm+zfbJwCs6G1ZExOSyW596TSvdDx8tu8vcWLb73AFs09mwIiImVy9W2q1qJZG/D9iY4lbST1BU40d2MqiIiMk2NFzd+yPHTOS2l5Q/PgS8o7PhRER0Ry82mbSqlV4rFzDKjUG2004eEX1juMLD2LbStHJCzc/TgNdT9GBpStJegG0vkbQbMBe4vuyWExHRU6o8HnkrTSv1T0G+WFLTm4UkfQw4CFhP0i+AvwEWAydK2sP2JycYb0RER/R708pWNS+nAHtSjA3QzBuA2cCGwB+BmbYflPQZirF3R03k5Qhi8wA22XAbpm2wxZgnEBHRDv3etHIFRRu5KJpUbgHeOcY2g7aHgEck3TQycpft1ZIaPl+jdkSxrTffucKfjxFRNX3dawXY1fYaA59L2nCMbR6XtLHtRygq+JHttgAq/IjTiOhXVa4cW/kIumSUeZeOsc0+ZRLHXuMJd+uTPugR0YOGrZanXtNsPPJnUDzVYqPyaRYj0W9OcYNQQ7YfazD/HuCeiYUaEdE5/dpr5VUUT7CYCZzOXxP5g8A/dTasiIjJVeU232bjkX8D+Iak15dDK0ZE9C2P+oS2amiljXxPSVuOvJA0XdI/dzCmiIhJN2i1PPWaVhL5Qbb/PPLC9v3AqzsXUkTE5DNqeeo1rXQ/nCppw5ELmJI2orjRJyKib/RlG3mNs4FfSvpa+fodwDc6F1JExOTrxUq7Va2MtfJpSVcD+1P0XPkZ8OxOBxYRMZn6vSKHYryUYeCNFLfopxdLRPSVoX6syCXtDBwOHAHcC3yX4rmd+01SbBERk6bCT3prWpFfD/wGeK3tlQCS3j8pUUVETLLhClfkzbofvp6iSeUCSV+W9Eqo8JlGRDThcUy9pmEit/0j228CnkfxUIj3A9tK+pKkAycpvoiISTE8jqnXjHlDkO2HbX/L9sEU464sA07seGQREZNoWGp56jXjGknd9n22z8iDlyOi3wyNY+o1rXY/jIjoa/3aayUi4imjyr1WejaRP/Dow90Ooe3+7U+jPWyp+q4a2q3bIbTdWTvf2+0QOuKg6/bqdgg9qxd7o7SqZxN5RMRkStNKRETF9WK3wlYlkUdEAEMVrsjH1f0wIqJftfOGIElzJd0gaaWkte67kXSMpGskLZN0kaTdapZ9uNzuBkmvaiX2JPKICNqXyCVNBeYDBwG7AUfUJurSt22/wPZs4NPA58ptd6MYrHB3YC7wxXJ/TSWRR0QAVuvTGPYCVtq+2fbjwDnAoWscy36w5uUm/LXTzKHAObYfs30LsLLcX1NpI4+IYHwXOyXNA+bVzFpge0H58wzg9pplA8DfjLKPY4HjgQ2AkbvlZwCX1W07Y6x4ksgjIhjfrfdl0l7QYPFoNfta3dRtzwfmS3oz8FHgyFa3rZdEHhFBW/uRDwDb1byeCaxqsv45wJcmuC2QNvKICKCtvVaWALMk7ShpA4qLlwtrV5A0q+bla4Aby58XAodL2lDSjsAs4PKxDpiKPCKC9t0QZHtQ0nHA+cBU4EzbyyWdAiy1vRA4TtL+wBPA/RTNKpTrfQ9YAQwCx9oes9UniTwigvaOtWJ7EbCobt5JNT+/t8m2nwQ+OZ7jJZFHRJCxViIiKq8XHxjRqiTyiAhguMID2SaRR0SQ0Q8jIiqvuvV4EnlEBJCKPCKi8gZV3Zo8iTwigmo3rUzaLfqSvjlZx4qIGK92PlhisnWkIpe0sH4WsJ+kLQFsH9KJ40ZETFS6H65tJsVYAV+h+MYiYA5werONasf4nTJ1C6ZM2aRD4UVErKm6abxzTStzgCuAjwAP2F4MrLZ9oe0LG21ke4HtObbnJIlHxGRK00od28PA5yV9v/z/nzp1rIiIdhiqcE3e0eRqewA4TNJrgAfHWj8iolt6sdJu1aRUybZ/Cvx0Mo4VETERTkUeEVFtqcgjIiou3Q8jIiquumk8iTwiAoDBCqfyJPKICHKxMyKi8nKxMyKi4lKRR0RUXCryiIiKG3Iq8oiISks/8oiIiksbeURExaWNPCKi4tK0EhFRcWlaiYiouPRaiYiouDStdEB139LGnhga7HYIHfHre1Z0O4S2O1ov6HYIHXHu1ad0O4SeVeWLnZ16+HJERKV4HP+NRdJcSTdIWinpxFGW7yPpSkmDkt5Qt2xI0rJyWthK7D1bkUdETKZ2Na1ImgrMBw4ABoAlkhbarv3q+gfgKOCEUXax2vbs8RwziTwiAnD7LnbuBay0fTOApHOAQ4EnE7ntW8tlbWnRSdNKRAQwhFueJM2TtLRmmlezqxnA7TWvB8p5rZpW7vMySa9rZYNU5BERjK9pxfYCYEGDxRptk3GEsr3tVZJ2An4l6RrbNzXbIBV5RARF00qr0xgGgO1qXs8EVo0jjlXl/28GFgN7jLVNEnlEBEVF3uo0hiXALEk7StoAOBxoqfeJpOmSNix/3hrYm5q29UaSyCMiaF/3Q9uDwHHA+cB1wPdsL5d0iqRDACS9WNIAcBhwhqTl5ea7AkslXQVcAJxW19tlVGkjj4igvbfo214ELKqbd1LNz0somlzqt7sEGPfdaEnkERHkFv2IiMpLIo+IqLg23hA06ZLIIyJIRR4RUXl5sERERMUNuboD2SaRR0SQNvKIiMpLG3lERMWljTwiouKG07TSnKSXUgy2fq3tn0/GMSMixqPKFXlHBs2SdHnNz0cD/w5sBnxstOfXRUR025CHW556Tacq8vVrfp4HHGD7bkmfBS4DThtto/IpG/MANHULpkzZpEPhRUSsKU0ra5siaTpFxS/bdwPYfljSYKONap+6sd4GM6r7rkZE5VS5aaVTiXwL4AqKRx5Z0jNs/1HSpoz+GKSIiK5KRV7H9g4NFg0Df9uJY0ZErItU5C2y/Qhwy2QeMyKiFUMe6nYIE5Z+5BER5Bb9iIjKyy36EREVl4o8IqLi0mslIqLi0mslIqLievHW+1YlkUdEkDbyiIjKSxt5RETFpSKPiKi49COPiKi4VOQRERWXXisRERWXi50RERWXppWIiIrLnZ0RERWXijwiouKq3EauKn8KtYukeeWDn/tKP55XP54T9Od59eM59aop3Q6gR8zrdgAd0o/n1Y/nBP15Xv14Tj0piTwiouKSyCMiKi6JvNCv7Xj9eF79eE7Qn+fVj+fUk3KxMyKi4lKRR0RUXBJ5RETFPaUTuaQzJd0l6dpux9IukraTdIGk6yQtl/TebsfUDpKmSbpc0lXleX282zG1i6Spkn4n6bxux9Iukm6VdI2kZZKWdjuefveUbiOXtA/wEPBN28/vdjztIOmZwDNtXylpM+AK4HW2V3Q5tHUiScAmth+StD5wEfBe25d1ObR1Jul4YA6wue2Dux1PO0i6FZhj+55ux/JU8JSuyG3/Griv23G0k+07bV9Z/vwX4DpgRnejWncuPFS+XL+cKl+FSJoJvAb4Srdjiep6SifyfidpB2AP4LfdjaQ9yiaIZcBdwC9s98N5fQH4EFDdpxqMzsDPJV0hKXd4dlgSeZ+StCnwA+B9th/sdjztYHvI9mxgJrCXpEo3h0k6GLjL9hXdjqUD9rb9IuAg4NiyGTM6JIm8D5VtyD8AvmX7h92Op91s/xlYDMztcijram/gkLI9+RzgFZLO7m5I7WF7Vfn/u4AfAXt1N6L+lkTeZ8qLgl8FrrP9uW7H0y6Sni5py/LnjYD9geu7G9W6sf1h2zNt7wAcDvzK9lu7HNY6k7RJeaEdSZsABwJ90zOsFz2lE7mk7wCXArtIGpD0zm7H1AZ7A2+jqO6WldOrux1UGzwTuEDS1cASijbyvumu12e2BS6SdBVwOfBT2z/rckx97Snd/TAioh88pSvyiIh+kEQeEVFxSeQRERWXRB4RUXFJ5BERFZdEHm0naajs9nitpO9L2ngd9rXvyKiAkg6RdGKTdbeU9A8TOMbJkk6YaIwR3ZZEHp2w2vbsckTJx4FjaheqMO6/PdsLbZ/WZJUtgXEn8oiqSyKPTvsN8FxJO5RjpH8RuBLYTtKBki6VdGVZuW8KIGmupOslXQT83ciOJB0l6d/Ln7eV9KNyfPKrJL0EOA14Tvlt4DPleh+UtETS1bVjmEv6iKQbJP1/YJdJezciOiCJPDpG0noUgyZdU87ahWLs9z2Ah4GPAvuXgystBY6XNA34MvBa4GXAMxrs/v8BF9p+IfAiYDlwInBT+W3gg5IOBGZRjPMxG9hT0j6S9qS4JX4Pig+KF7f51CMm1XrdDiD60kblcLNQVORfBZ4F3FbzIIj/CewGXFwMD8MGFMMlPA+4xfaNAOUgUqMNg/oK4O1QjIoIPCBpet06B5bT78rXm1Ik9s2AH9l+pDzGwnU624guSyKPTlhdDjf7pDJZP1w7i2K8lCPq1ptN+x4YIeBU22fUHeN9bTxGRNelaSW65TJgb0nPBZC0saSdKUY03FHSc8r1jmiw/S+B/1NuO1XS5sBfKKrtEecDf1/T9j5D0jbAr4G/lbRROUrfa9t8bhGTKok8usL23cBRwHfKEQ0vA55n+1GKppSflhc7b2uwi/cC+0m6huK5pLvbvpeiqeZaSZ+x/XPg28Cl5XrnApuVj8L7LrCMYtz233TsRCMmQUY/jIiouFTkEREVl0QeEVFxSeQRERWXRB4RUXFJ5BERFZdEHhFRcUnkEREV99+KVjRYT5ff3wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x219482f5668>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.figure.Figure at 0x21949ce3f60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "all_preds = np.array(all_preds)\n",
    "print(all_preds.shape)\n",
    "y_predictions = all_preds.argmax(axis=1)\n",
    "y_true = y_test.argmax(axis=1)\n",
    "y_true = y_true[:len(y_predictions)]\n",
    "\n",
    "cm = ConfusionMatrix(y_true, y_predictions)\n",
    "cm.plot(backend='seaborn', normalized=True)\n",
    "plt.title('Confusion Matrix Stars prediction')\n",
    "plt.figure(figsize=(12, 10))\n",
    "\n",
    "test_correctPred = np.equal(y_predictions, y_true)\n",
    "test_accuracy = np.mean(test_correctPred.astype(float))\n",
    "\n",
    "print(test_accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test out the network yourself!\n",
    "User can enter a review here and see how the network does in predicting his or her review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Please enter a review in englishI love this place! Will definitely come back!\n",
      "INFO:tensorflow:Restoring parameters from ./saves/best_model.ckpt\n",
      "\n",
      "You rated the restaurant: 5 stars!\n"
     ]
    }
   ],
   "source": [
    "load_files = True\n",
    "if load_files == True:\n",
    "    word_embedding_matrix = loadfiles(\"./data/pickles/word_embedding_matrix.p\")\n",
    "    word2int = loadfiles('./data/pickles/word2int.p')\n",
    "\n",
    "pred_text = input(\"Please enter a review in english\")\n",
    "contractions = get_contractions()\n",
    "pred_text = clean_text(pred_text)\n",
    "pred_seq = convert_to_ints(pred_text, pred=True)\n",
    "pred_seq = np.tile(pred_seq, (batch_size, 1))\n",
    "\n",
    "with tf.Session(graph=train_graph) as sess:\n",
    "    checkpoint = \"./saves/best_model.ckpt\"  \n",
    "    all_preds = []\n",
    "    with tf.Session() as sess:\n",
    "        saver = tf.train.Saver()\n",
    "        # Load the model\n",
    "        saver.restore(sess, checkpoint)\n",
    "        test_state = sess.run(graph.initial_state)\n",
    "        feed = {graph.input_data: pred_seq,\n",
    "                graph.keep_prob: keep_probability,\n",
    "                graph.initial_state: state}\n",
    "\n",
    "        predictions = sess.run(graph.predictions, feed_dict=feed)\n",
    "        for i in range(len(predictions)):\n",
    "            all_preds.append(predictions[i,:])\n",
    "all_preds = np.array(all_preds)\n",
    "y_predictions = all_preds.argmax(axis=1)\n",
    "counts = np.bincount(y_predictions)\n",
    "print(\"\\nYou rated the restaurant: \" + str(np.argmax(counts)) + \" stars!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}