files/nlp23/week_09/17_dit_nlp_brnn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bidirectional recurrent neural networks\n",
    "\n",
    "In this notebook, we produce a recurrent neural network that considers both the outputs at time $t-1$ from the left and from the right."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing the dependencies\n",
    "import glob\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "from gensim.models.keyedvectors import KeyedVectors\n",
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "from nlpia.loaders import get_data\n",
    "from random import shuffle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add the paths to the corpus. It should end in aclImdb/train\n",
    "CORPUS_PATH = None\n",
    "# Add the path to the embeddings. It should end in GoogleNews-vectors-negative300.bin.gz\n",
    "GOOGLE_VECTORS = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data preprocessor\n",
    "def pre_process_data(filepath):\n",
    "    \"\"\"\n",
    "    Load pos and neg examples from separate dirs then shuffle them\n",
    "    together.\n",
    "    \"\"\"\n",
    "    positive_path = os.path.join(filepath, 'pos')\n",
    "    negative_path = os.path.join(filepath, 'neg')\n",
    "    pos_label = 1\n",
    "    neg_label = 0\n",
    "    dataset = []\n",
    "    for filename in glob.glob(os.path.join(positive_path, '*.txt')):\n",
    "        with open(filename, 'r') as f:\n",
    "            dataset.append((pos_label, f.read()))\n",
    "    for filename in glob.glob(os.path.join(negative_path, '*.txt')):\n",
    "        with open(filename, 'r') as f:\n",
    "            dataset.append((neg_label, f.read()))\n",
    "    shuffle(dataset)\n",
    "    return dataset\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tokenizing and vectorizing all the instances\n",
    "def tokenize_and_vectorize(dataset):\n",
    "    tokenizer = TreebankWordTokenizer()\n",
    "    vectorized_data = []\n",
    "    for sample in dataset:\n",
    "        tokens = tokenizer.tokenize(sample[1])\n",
    "        sample_vecs = []\n",
    "        for token in tokens:\n",
    "            try:\n",
    "                sample_vecs.append(word_vectors[token])\n",
    "            except KeyError:\n",
    "                pass\n",
    "        vectorized_data.append(sample_vecs)\n",
    "    return vectorized_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Method to pad or truncate the input\n",
    "# Not necessary in general; we apply it for comparison against the \n",
    "# previous session\n",
    "def pad_trunc(data, maxlen):\n",
    "    \"\"\"\n",
    "    For a given dataset pad with zero vectors or truncate to maxlen\n",
    "    \"\"\"\n",
    "    new_data = []\n",
    "    # Create a vector of 0s the length of our word vectors\n",
    "    zero_vector = []\n",
    "    for _ in range(len(data[0][0])):\n",
    "        zero_vector.append(0.0)\n",
    "\n",
    "    for sample in data:\n",
    "        if len(sample) > maxlen:\n",
    "            temp = sample[:maxlen]\n",
    "        elif len(sample) < maxlen:\n",
    "            temp = sample\n",
    "            # Append the appropriate number 0 vectors to the list\n",
    "            additional_elems = maxlen - len(sample)\n",
    "            for _ in range(additional_elems):\n",
    "                temp.append(zero_vector)\n",
    "        else:\n",
    "            temp = sample\n",
    "        new_data.append(temp)\n",
    "\n",
    "    return new_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extracting the expected output for all the instances\n",
    "\n",
    "# (this is so verbose!)\n",
    "# def collect_expected(dataset):\n",
    "#     \"\"\" Peel off the target values from the dataset \"\"\"\n",
    "#     expected = []\n",
    "#     for sample in dataset:\n",
    "#         expected.append(sample[0])\n",
    "#     return expected\n",
    "\n",
    "def collect_expected(dataset):\n",
    "    return [sample[0] for sample in dataset]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading the embeddings\n",
    "word_vectors = KeyedVectors.load_word2vec_format(GOOGLE_VECTORS,\n",
    "    binary=True, limit=400000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data preparation\n",
    "dataset = pre_process_data(CORPUS_PATH)\n",
    "vectorized_data = tokenize_and_vectorize(dataset)\n",
    "expected = collect_expected(dataset)\n",
    "split_point = int(len(vectorized_data) * .8)\n",
    "\n",
    "x_train = vectorized_data[:split_point]\n",
    "y_train = expected[:split_point]\n",
    "\n",
    "x_test = vectorized_data[split_point:]\n",
    "y_test = expected[split_point:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Network parameters\n",
    "maxlen = 400         \n",
    "batch_size = 32      \n",
    "embedding_dims = 300\n",
    "epochs = 2\n",
    "num_neurons = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading data\n",
    "\n",
    "x_train = pad_trunc(x_train, maxlen)\n",
    "x_test = pad_trunc(x_test, maxlen)\n",
    "\n",
    "x_train = np.reshape(x_train, (len(x_train), maxlen, embedding_dims))\n",
    "y_train = np.array(y_train)\n",
    "\n",
    "x_test = np.reshape(x_test, (len(x_test), maxlen, embedding_dims))\n",
    "y_test = np.array(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing the dependencies\n",
    "from keras.models import Sequential\n",
    "from keras.layers.wrappers import Bidirectional          # <- this is the new wrapper\n",
    "from keras.layers import Dense, Dropout, Flatten, SimpleRNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "# Adding one bidirectional recurrent layer\n",
    "model.add(Bidirectional(SimpleRNN(\n",
    "    num_neurons, \n",
    "    return_sequences=True),\n",
    "    input_shape=(maxlen, embedding_dims))\n",
    "    )\n",
    "\n",
    "# Compare it against what we had last time\n",
    "# model.add(SimpleRNN(\n",
    "#     num_neurons, \n",
    "#     return_sequences=True,\n",
    "#     input_shape=(maxlen, embedding_dims))\n",
    "#      )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adding a dropout layer (remember why?)\n",
    "model.add(Dropout(.2))\n",
    "\n",
    "# Adding a flattening layer\n",
    "model.add(Flatten())\n",
    "\n",
    "# Adding the classifier\n",
    "model.add(Dense(1, activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compiling the network\n",
    "model.compile('rmsprop', \n",
    "              'binary_crossentropy', \n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training the network\n",
    "\n",
    "model.fit(x_train, y_train,\n",
    "    batch_size=batch_size,\n",
    "    epochs=epochs,\n",
    "    validation_data=(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predicting\n",
    "sample_1 = \"\"\"I hate that the dismal weather had me down for so long, when\n",
    "will it break! Ugh, when does happiness return? The sun is blinding and\n",
    "the puffy clouds are too thin. I can't wait for the weekend.\"\"\"\n",
    "\n",
    "vec_list = tokenize_and_vectorize([(1, sample_1)])\n",
    "test_vec_list = pad_trunc(vec_list, maxlen)\n",
    "test_vec = np.reshape(\n",
    "    test_vec_list, \n",
    "    (len(test_vec_list), maxlen, embedding_dims))\n",
    "model.predict_classes(test_vec)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}