runnable_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/yashbidasaria/nmt/blob/master/runnable_model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Execute these cells only to download the pickle files\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 211
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "MhwsBjR5sW6b",
    "outputId": "ef781094-e678-4918-f777-bbcb5133a62f"
   },
   "outputs": [],
   "source": [
    "!pip install PyDrive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": false,
    "id": "LqWczR5VtDOm"
   },
   "outputs": [],
   "source": [
    "from pydrive.auth import GoogleAuth\n",
    "from pydrive.drive import GoogleDrive\n",
    "#from google.colab import auth\n",
    "from oauth2client.client import GoogleCredentials\n",
    "# Authenticate and create the PyDrive client.\n",
    "# This only needs to be done once per notebook.\n",
    "#auth.authenticate_user()\n",
    "gauth = GoogleAuth()\n",
    "gauth.credentials = GoogleCredentials.get_application_default()\n",
    "drive = GoogleDrive(gauth)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "!sudo {sys.executable} -m pip install pydrive\n",
    "!sudo {sys.executable} -m pip install --upgrade google-api-python-client google-auth-httplib2 google-auth-oauthlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "german = drive.CreateFile({'id': '1UqJmu1fvDyNWx2D7joWY9aAmeKRY9tw6'})\n",
    "german_name = german['title']\n",
    "german.GetContentFile(german_name)\n",
    "\n",
    "english = drive.CreateFile({'id': '1Oeo5SwHEbSKhObxTl2RZk2_jhKVxBVqZ'})\n",
    "english_name = english['title']\n",
    "english.GetContentFile(english_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model Code "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "vehlG055vOBr"
   },
   "outputs": [],
   "source": [
    "\n",
    "import pickle, torch, time, random\n",
    "from torch import nn, optim\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": false,
    "id": "BuFQMXp4sjwd"
   },
   "outputs": [],
   "source": [
    "german_name = \"german.pickle\"\n",
    "english_name = \"english.pickle\"\n",
    "german_data = {}\n",
    "with open(german_name, \"rb\") as f:\n",
    "    german_data = pickle.load(f)\n",
    "english_data = {}\n",
    "with open(english_name, \"rb\") as f:\n",
    "    english_data = pickle.load(f)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have to concetinate the hidden states of the encoder with the states of the decoder, therefore the length has to be of the same size, can figure out a better non-hacky way later. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pad = 768 - 743\n",
    "pad_arr = [0]*pad\n",
    "idx = 0\n",
    "for i in english_data['train']:\n",
    "    i = i + pad_arr\n",
    "    english_data['train'][idx] = i\n",
    "    idx += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "jaUMYYIQtkGL"
   },
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "english_data['train'] = torch.LongTensor(english_data['train']).cuda()\n",
    "english_data['dev'] = torch.LongTensor(english_data['dev']).cuda()\n",
    "german_data['train'] = torch.LongTensor(german_data['train']).cuda()\n",
    "german_data['dev'] = torch.LongTensor(german_data['dev']).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "AXjpf1mK23kS"
   },
   "outputs": [],
   "source": [
    "def train(eng_train, eng_dev, de_train, de_dev, net, params):\n",
    "  \n",
    "    # padding is 0\n",
    "    criterion = nn.CrossEntropyLoss(ignore_index=0)\n",
    "    \n",
    "    optimizer = optim.Adam(net.parameters(), lr=params['learning_rate'])\n",
    "    \n",
    "    num_examples, eng_len = eng_train.size()    \n",
    "    batches = [(start, start + params['batch_size']) for start in\\\n",
    "               range(0, num_examples, params['batch_size'])]\n",
    "    \n",
    "    \n",
    "    for epoch in range(params['epochs']):\n",
    "        ep_loss = 0.\n",
    "        start_time = time.time()\n",
    "        random.shuffle(batches)\n",
    "        \n",
    "        # for each batch, calculate loss and optimize model parameters            \n",
    "        for b_idx, (start, end) in enumerate(batches):\n",
    "            \n",
    "            de_src = de_train[start:end]\n",
    "            eng_trg = eng_train[start:end]\n",
    "            print(\"de_src size: \", de_src.size())\n",
    "            preds = net(de_src, eng_trg)\n",
    "            \n",
    "            # q1.1: explain the below line!\n",
    "            preds = preds[1:].view(-1, preds.shape[-1])\n",
    "            print(\"preds size \", preds.size())\n",
    "            eng_trg = eng_trg[1:].view(-1)\n",
    "            print(\"trg size \", eng_trg.size())\n",
    "            loss = criterion(preds, eng_trg)\n",
    "            \n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "            ep_loss += loss\n",
    "\n",
    "        print('epoch: {0}, loss: {1}, time: {2}'.format(epoch, ep_loss, time.time()-start_time))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "X0XynGos6u5p"
   },
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, input_dim, emb_dim, enc_hid_dim, dec_hid_dim, dropout):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.input_dim = input_dim\n",
    "        self.emb_dim = emb_dim\n",
    "        self.enc_hid_dim = enc_hid_dim\n",
    "        self.dec_hid_dim = dec_hid_dim\n",
    "        self.dropout = dropout\n",
    "        \n",
    "        self.embedding = nn.Embedding(input_dim, emb_dim)\n",
    "        \n",
    "        self.rnn = nn.GRU(emb_dim, enc_hid_dim, bidirectional=True)\n",
    "        \n",
    "        self.fc = nn.Linear(enc_hid_dim * 2, dec_hid_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, src):\n",
    "        \n",
    "        #src = [src sent len, batch size]\n",
    "        \n",
    "        embedded = self.dropout(self.embedding(src))\n",
    "        \n",
    "        #embedded = [src sent len, batch size, emb dim]\n",
    "        \n",
    "        outputs, hidden = self.rnn(embedded)\n",
    "                \n",
    "        #outputs = [src sent len, batch size, hid dim * num directions]\n",
    "        #hidden = [n layers * num directions, batch size, hid dim]\n",
    "        \n",
    "        #hidden is stacked [forward_1, backward_1, forward_2, backward_2, ...]\n",
    "        #outputs are always from the last layer\n",
    "        \n",
    "        #hidden [-2, :, : ] is the last of the forwards RNN \n",
    "        #hidden [-1, :, : ] is the last of the backwards RNN\n",
    "        \n",
    "        #initial decoder hidden is final hidden state of the forwards and backwards encoder RNNs fed through a linear layer\n",
    "        hidden = torch.tanh(self.fc(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim=1)))\n",
    "        \n",
    "        #outputs = [src sent len, batch size, enc hid dim * 2]\n",
    "        #hidden = [batch size, dec hid dim]\n",
    "        \n",
    "        return outputs, hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "ombD8ddo7H3H"
   },
   "outputs": [],
   "source": [
    "class Attention(nn.Module):\n",
    "    def __init__(self, enc_hid_dim, dec_hid_dim):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.enc_hid_dim = enc_hid_dim\n",
    "        self.dec_hid_dim = dec_hid_dim\n",
    "        \n",
    "        self.attn = nn.Linear((enc_hid_dim * 2) + dec_hid_dim, dec_hid_dim)\n",
    "        self.v = nn.Parameter(torch.rand(dec_hid_dim))\n",
    "        \n",
    "    def forward(self, hidden, encoder_outputs):\n",
    "        \n",
    "        #hidden = [batch size, dec hid dim]\n",
    "        #encoder_outputs = [src sent len, batch size, enc hid dim * 2]\n",
    "        \n",
    "        batch_size = encoder_outputs.shape[1]\n",
    "        src_len = encoder_outputs.shape[0]\n",
    "        \n",
    "        #repeat encoder hidden state src_len times\n",
    "        hidden = hidden.unsqueeze(1).repeat(1, src_len, 1)\n",
    "        \n",
    "        encoder_outputs = encoder_outputs.permute(1, 0, 2)\n",
    "        \n",
    "        #hidden = [batch size, src sent len, dec hid dim]\n",
    "        #encoder_outputs = [batch size, src sent len, enc hid dim * 2]\n",
    "        \n",
    "        energy = torch.tanh(self.attn(torch.cat((hidden, encoder_outputs), dim=2))) \n",
    "        \n",
    "        #energy = [batch size, src sent len, dec hid dim]\n",
    "        \n",
    "        energy = energy.permute(0, 2, 1)\n",
    "        \n",
    "        #energy = [batch size, dec hid dim, src sent len]\n",
    "        \n",
    "        #v = [dec hid dim]\n",
    "        \n",
    "        v = self.v.repeat(batch_size, 1).unsqueeze(1)\n",
    "        \n",
    "        #v = [batch size, 1, dec hid dim]\n",
    "                \n",
    "        attention = torch.bmm(v, energy).squeeze(1)\n",
    "        \n",
    "        #attention= [batch size, src len]\n",
    "        \n",
    "        return torch.softmax(attention, dim=1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "gAlFVF5k7Ku_"
   },
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, output_dim, emb_dim, enc_hid_dim, dec_hid_dim, dropout, attention):\n",
    "        super().__init__()\n",
    "\n",
    "        self.emb_dim = emb_dim\n",
    "        self.enc_hid_dim = enc_hid_dim\n",
    "        self.dec_hid_dim = dec_hid_dim\n",
    "        self.output_dim = output_dim\n",
    "        self.dropout = dropout\n",
    "        self.attention = attention\n",
    "        \n",
    "        self.embedding = nn.Embedding(output_dim, emb_dim)\n",
    "        \n",
    "        self.rnn = nn.GRU((enc_hid_dim * 2) + emb_dim, dec_hid_dim)\n",
    "        \n",
    "        self.out = nn.Linear((enc_hid_dim * 2) + dec_hid_dim + emb_dim, output_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, input, hidden, encoder_outputs):\n",
    "             \n",
    "        #input = [batch size]\n",
    "        #hidden = [batch size, dec hid dim]\n",
    "        #encoder_outputs = [src sent len, batch size, enc hid dim * 2]\n",
    "        \n",
    "        input = input.unsqueeze(0)\n",
    "        \n",
    "        #input = [1, batch size]\n",
    "        \n",
    "        embedded = self.dropout(self.embedding(input))\n",
    "        \n",
    "        #embedded = [1, batch size, emb dim]\n",
    "        \n",
    "        a = self.attention(hidden, encoder_outputs)\n",
    "                \n",
    "        #a = [batch size, src len]\n",
    "        \n",
    "        a = a.unsqueeze(1)\n",
    "        \n",
    "        #a = [batch size, 1, src len]\n",
    "        \n",
    "        encoder_outputs = encoder_outputs.permute(1, 0, 2)\n",
    "        \n",
    "        #encoder_outputs = [batch size, src sent len, enc hid dim * 2]\n",
    "        \n",
    "        weighted = torch.bmm(a, encoder_outputs)\n",
    "        \n",
    "        #weighted = [batch size, 1, enc hid dim * 2]\n",
    "        \n",
    "        weighted = weighted.permute(1, 0, 2)\n",
    "        \n",
    "        #weighted = [1, batch size, enc hid dim * 2]\n",
    "        \n",
    "        rnn_input = torch.cat((embedded, weighted), dim=2)\n",
    "        \n",
    "        #rnn_input = [1, batch size, (enc hid dim * 2) + emb dim]\n",
    "            \n",
    "        output, hidden = self.rnn(rnn_input, hidden.unsqueeze(0))\n",
    "        \n",
    "        #output = [sent len, batch size, dec hid dim * n directions]\n",
    "        #hidden = [n layers * n directions, batch size, dec hid dim]\n",
    "        \n",
    "        #sent len, n layers and n directions will always be 1 in this decoder, therefore:\n",
    "        #output = [1, batch size, dec hid dim]\n",
    "        #hidden = [1, batch size, dec hid dim]\n",
    "        #this also means that output == hidden\n",
    "        assert (output == hidden).all()\n",
    "        \n",
    "        embedded = embedded.squeeze(0)\n",
    "        output = output.squeeze(0)\n",
    "        weighted = weighted.squeeze(0)\n",
    "        \n",
    "        output = self.out(torch.cat((output, weighted, embedded), dim=1))\n",
    "        \n",
    "        #output = [bsz, output dim]\n",
    "        \n",
    "        return output, hidden.squeeze(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "Ka_pvZC07N9M"
   },
   "outputs": [],
   "source": [
    "class Base(nn.Module):\n",
    "    def __init__(self, encoder, decoder, device):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        self.device = device\n",
    "        \n",
    "    def forward(self, src, trg, teacher_forcing_ratio=0.5):\n",
    "        \n",
    "        #src = [src sent len, batch size]\n",
    "        #trg = [trg sent len, batch size]\n",
    "        #teacher_forcing_ratio is probability to use teacher forcing\n",
    "        #e.g. if teacher_forcing_ratio is 0.75 we use teacher forcing 75% of the time\n",
    "        \n",
    "        batch_size = src.shape[1]\n",
    "        max_len = trg.shape[0]\n",
    "        trg_vocab_size = self.decoder.output_dim\n",
    "        \n",
    "        #tensor to store decoder outputs\n",
    "        outputs = torch.zeros(max_len, batch_size, trg_vocab_size).to(self.device)\n",
    "        \n",
    "        #encoder_outputs is all hidden states of the input sequence, back and forwards\n",
    "        #hidden is the final forward and backward hidden states, passed through a linear layer\n",
    "        encoder_outputs, hidden = self.encoder(src)\n",
    "                \n",
    "        #first input to the decoder is the <sos> tokens\n",
    "        output = trg[0,:]\n",
    "        \n",
    "        for t in range(1, max_len):\n",
    "            output, hidden = self.decoder(output, hidden, encoder_outputs)\n",
    "            outputs[t] = output\n",
    "            teacher_force = random.random() < teacher_forcing_ratio\n",
    "            top1 = output.max(1)[1]\n",
    "            output = (trg[t] if teacher_force else top1)\n",
    "\n",
    "        return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": false,
    "id": "rZKeFe3n7CcG"
   },
   "outputs": [],
   "source": [
    "INPUT_DIM = len(german_data['idx2word'])\n",
    "OUTPUT_DIM = len(english_data['idx2word'])\n",
    "ENC_EMB_DIM = 256\n",
    "DEC_EMB_DIM = 256\n",
    "ENC_HID_DIM = 512\n",
    "DEC_HID_DIM = 512\n",
    "ENC_DROPOUT = 0.5\n",
    "DEC_DROPOUT = 0.5\n",
    "\n",
    "attn = Attention(ENC_HID_DIM, DEC_HID_DIM)\n",
    "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, ENC_DROPOUT)\n",
    "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, DEC_DROPOUT, attn)\n",
    "\n",
    "model = Base(enc, dec, device).to(device)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": false,
    "id": "BcMhQAk69a09"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "de_src size:  torch.Size([16, 768])\n",
      "preds size  torch.Size([11520, 58510])\n",
      "trg size  torch.Size([11520])\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "CUDA out of memory. Tried to allocate 2.51 GiB (GPU 0; 11.17 GiB total capacity; 8.94 GiB already allocated; 1.83 GiB free; 126.43 MiB cached)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-8149c86cf3df>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'learning_rate'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.001\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0menglish_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'train'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m256\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menglish_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'dev'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m256\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgerman_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'train'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m256\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgerman_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'dev'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m256\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-5-6974f63173f9>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(eng_train, eng_dev, de_train, de_dev, net, params)\u001b[0m\n\u001b[1;32m     31\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meng_trg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m             \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m             \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m             \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    100\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m         \"\"\"\n\u001b[0;32m--> 102\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    104\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     88\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     89\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: CUDA out of memory. Tried to allocate 2.51 GiB (GPU 0; 11.17 GiB total capacity; 8.94 GiB already allocated; 1.83 GiB free; 126.43 MiB cached)"
     ]
    }
   ],
   "source": [
    "params = {}\n",
    "params['batch_size'] = 16\n",
    "params['epochs'] = 1\n",
    "params['learning_rate'] = 0.001\n",
    "\n",
    "train(english_data['train'][:256], english_data['dev'][:256], german_data['train'][:256], german_data['dev'][:256], model, params)\n"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "include_colab_link": true,
   "name": "runnable_model.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}