Deep_Learning_Final_Project.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VFKW-B7DBhWs",
    "colab_type": "text"
   },
   "source": [
    "## Fetch Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "eTBc5W1r_ez0",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "!rm *.py\n",
    "!git clone https://github.com/tanmaybinaykiya/CS-7643-Deep-Learning-Final-Project.git \n",
    "!mv CS-7643-Deep-Learning-Final-Project/*.py .\n",
    "!rm -rf 'CS-7643-Deep-Learning-Final-Project'\n",
    "!mkdir data\n",
    "!mkdir dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XjLpkRPDH-Tg",
    "colab_type": "text"
   },
   "source": [
    "# Install Libraries"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3ooZIyL4IViP",
    "colab_type": "text"
   },
   "source": [
    "## Install Pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "SHRAYI5xH9Ch",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "# http://pytorch.org/\n",
    "from os import path\n",
    "from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag\n",
    "platform = '{}{}-{}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag())\n",
    "\n",
    "accelerator = 'cu80' if path.exists('/opt/bin/nvidia-smi') else 'cpu'\n",
    "print(\"platform, accelerator:\", platform, accelerator)\n",
    "!pip install -v -q http://download.pytorch.org/whl/{accelerator}/torch-0.4.0-{platform}-linux_x86_64.whl torchvision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "U5Gv1TQgjM8P",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.nn.utils import clip_grad_norm_\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "from constants import DatasetPaths\n",
    "\n",
    "from DataLoader import SquadDataset, collate_fn, GloVeEmbeddings\n",
    "from models import EncoderBILSTM, DecoderLSTM\n",
    "from train import train, greedy_search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YaC4zPb0Bm9m",
    "colab_type": "text"
   },
   "source": [
    "# Data Processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RQBz1oPtEMMx",
    "colab_type": "text"
   },
   "source": [
    "### Download GloVe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "3plbto4sATqB",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "!wget 'http://nlp.stanford.edu/data/glove.840B.300d.zip'\n",
    "!unzip 'glove.840B.300d.zip'\n",
    "!rm glove.840B.300d.zip\n",
    "!mv glove.840B.300d.txt data/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CCs2uHw_E4Eg",
    "colab_type": "text"
   },
   "source": [
    "### Download Squad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "uQW1WAxfE6hh",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "!mkdir dataset\n",
    "!curl -o 'dataset/squad-train-v1.1.json' 'https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json' \n",
    "!curl -o 'dataset/squad-dev-v1.1.json' 'https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json' "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ApeYvHvdEr1z",
    "colab_type": "text"
   },
   "source": [
    "### Data Preprocessor\n",
    "\n",
    "- Builds question answer pairs: (Question: [index], Answer [index], Paragraph [index])\n",
    "- Builds a map of paragraphs: {index: paragraph}\n",
    "- Builds a word_to_idx map for questions and answers: {word: index}\n",
    "- Builds an idx_to_word map for questions and answers: {index: word}\n",
    "- Prunes glove embeddings for questions and answers: {word:embedding}\n",
    "- Maintains the most frequent 45000 words in answers and 28000 words in questions, all others are stored as <unk>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "4xZa4VlZjGjU",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "from DataProcessor import SquadPreProcessor, GlovePreproccesor\n",
    "\n",
    "train_ds = SquadPreProcessor(path=DatasetPaths[\"squad\"][\"train\"], split=\"train\", q_vocab_size=45000, a_vocab_size=28000)\n",
    "paragraphs, question_answer_pairs = train_ds.preprocess()\n",
    "train_ds.persist(paragraphs, question_answer_pairs)\n",
    "\n",
    "dev_ds = SquadPreProcessor(path=DatasetPaths[\"squad\"][\"dev\"], split=\"dev\", q_vocab_size=45000, a_vocab_size=28000)\n",
    "paragraphs, question_answer_pairs = dev_ds.preprocess()\n",
    "dev_ds.persist(paragraphs, question_answer_pairs)\n",
    "\n",
    "GlovePreproccesor().obtain_glove_embeddings(glove_filename=DatasetPaths[\"glove\"][\"original-embeddings\"],\n",
    "                                            word_to_ix=train_ds.a_word_to_idx,\n",
    "                                            pruned_glove_filename=DatasetPaths[\"glove\"][\"answer-embeddings\"])\n",
    "\n",
    "GlovePreproccesor().obtain_glove_embeddings(glove_filename=DatasetPaths[\"glove\"][\"original-embeddings\"],\n",
    "                                            word_to_ix=train_ds.q_word_to_idx,\n",
    "                                            pruned_glove_filename=DatasetPaths[\"glove\"][\"question-embeddings\"])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mo2GBaSwFWiN",
    "colab_type": "text"
   },
   "source": [
    "# Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "4rj8LUuihHKZ",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline  \n",
    "import numpy as np\n",
    "\n",
    "def plot_losses(losses):\n",
    "  plt.plot(losses)\n",
    "\n",
    "  plt.xlabel('Epoch')\n",
    "  plt.ylabel('Loss')\n",
    "  plt.title('Loss vs Epoch')\n",
    "  plt.grid(True)\n",
    "\n",
    "  plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wJGRpcwh6I1W",
    "colab_type": "text"
   },
   "source": [
    "### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "g6zuBz9pFTZo",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "use_cuda = use_cuda and torch.cuda.is_available()\n",
    "\n",
    "train_dataset = SquadDataset(split=\"train\")\n",
    "word_to_idx_sent = train_dataset.get_answer_word_to_idx()\n",
    "word_to_idx_q = train_dataset.get_question_idx_to_word()\n",
    "\n",
    "train_vocab_size_sent = len(word_to_idx_sent)\n",
    "train_vocab_size_q = len(word_to_idx_q)\n",
    "num_epoch = 15\n",
    "batch_size = 64\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=0, collate_fn=collate_fn, pin_memory=True)\n",
    "\n",
    "word_embeddings_glove_q = GloVeEmbeddings.load_glove_embeddings(True)\n",
    "word_embeddings_glove_sent = GloVeEmbeddings.load_glove_embeddings(False)\n",
    "\n",
    "encoder = EncoderBILSTM(vocab_size=train_vocab_size_sent, n_layers=2, embedding_dim=300, hidden_dim=500, dropout=0, embeddings=word_embeddings_glove_sent)\n",
    "decoder = DecoderLSTM(vocab_size=train_vocab_size_q, embedding_dim=300, hidden_dim=500, n_layers=1, encoder_hidden_dim=500, embeddings=word_embeddings_glove_q)\n",
    "\n",
    "if use_cuda:\n",
    "    encoder = encoder.cuda()\n",
    "    decoder = decoder.cuda()\n",
    "\n",
    "n_train = len(train_loader)\n",
    "batch_per_epoch = n_train // batch_size\n",
    "\n",
    "criterion = nn.CrossEntropyLoss(ignore_index=0)\n",
    "optimizer_enc = torch.optim.RMSprop(encoder.parameters(), lr=1, momentum=0.7)\n",
    "optimizer_dec = torch.optim.RMSprop(decoder.parameters(), lr=1, momentum=0.7)\n",
    "\n",
    "if not os.path.isdir(\"model_weights\"):\n",
    "    os.makedirs(\"model_weights\", exist_ok=True)\n",
    "\n",
    "losses= train(encoder=encoder, decoder=decoder, epoch_count=num_epoch, batch_per_epoch=batch_per_epoch, idx_to_word_q=None,\n",
    "                   train_loader=train_loader, criterion=criterion, optimizer_enc=optimizer_enc, optimizer_dec=optimizer_dec, \n",
    "                   is_cuda=use_cuda, debug=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "EoZwNJ5teCld",
    "colab_type": "text"
   },
   "source": [
    "### Visualize Loss Profile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "cRQ74JUF9Whj",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "plot_losses(losses)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hUs38HapeGez",
    "colab_type": "text"
   },
   "source": [
    "### Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "79N4KGVZBpXU",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "def predict():\n",
    "  predict_batch_size = 10\n",
    "  \n",
    "  dev_dataset = SquadDataset(split=\"dev\")\n",
    "\n",
    "  dev_loader = DataLoader(\n",
    "      dev_dataset, batch_size=predict_batch_size, shuffle=True, num_workers=0, collate_fn=collate_fn, pin_memory=True)\n",
    "  dev_idx_to_word_q = dev_dataset.get_question_idx_to_word()\n",
    "  dev_idx_to_word_sent = dev_dataset.get_answer_idx_to_word()\n",
    "\n",
    "  encoder = EncoderBILSTM(vocab_size=train_vocab_size_sent, n_layers=2, embedding_dim=300, hidden_dim=500, dropout=0, embeddings=word_embeddings_glove_sent)\n",
    "  decoder = DecoderLSTM(vocab_size=train_vocab_size_q, embedding_dim=300, hidden_dim=500, n_layers=1, encoder_hidden_dim=500, embeddings=word_embeddings_glove_q)\n",
    "  if use_cuda:\n",
    "    encoder.cuda()\n",
    "    decoder.cuda()\n",
    "  encoder.load_state_dict(torch.load(\"model_weights/1-encoder.pth\"))\n",
    "  decoder.load_state_dict(torch.load(\"model_weights/1-decoder.pth\"))\n",
    "\n",
    "  idx_to_word_sent = train_dataset.get_answer_idx_to_word()\n",
    "  idx_to_word_q = train_dataset.get_question_idx_to_word()\n",
    "\n",
    "  greedy_search(encoder, decoder, train_loader, True, idx_to_word_q, idx_to_word_sent, batch_size=predict_batch_size)\n",
    "  greedy_search(encoder, decoder, dev_loader, True, dev_idx_to_word_q, dev_idx_to_word_sent, batch_size=predict_batch_size)\n",
    "  \n",
    "predict()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "Deep Learning Final Project",
   "version": "0.3.2",
   "provenance": [],
   "collapsed_sections": [
    "RQBz1oPtEMMx",
    "CCs2uHw_E4Eg"
   ],
   "toc_visible": true
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "accelerator": "GPU"
 },
 "nbformat": 4,
 "nbformat_minor": 0
}