pytorch_ipynb/rnn/char_rnn-charlesdickens.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
    "- Author: Sebastian Raschka\n",
    "- GitHub Repository: https://github.com/rasbt/deeplearning-models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vY4SK0xKAJgm"
   },
   "source": [
    "# Model Zoo -- Character RNN for Generating Text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "L24QVN8uwirR"
   },
   "source": [
    "A simple character-level RNN to generate new bits of text based on text from a novel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "moNmVfuvnImW"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sebastian Raschka \n",
      "\n",
      "CPython 3.7.1\n",
      "IPython 7.4.0\n",
      "\n",
      "torch 1.0.1.post2\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -a 'Sebastian Raschka' -v -p torch\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torchtext import data\n",
    "from torchtext import datasets\n",
    "import time\n",
    "import random\n",
    "import unidecode\n",
    "import string\n",
    "import random\n",
    "import re\n",
    "\n",
    "\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GSRL42Qgy8I8"
   },
   "source": [
    "## General Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OvW1RgfepCBq"
   },
   "outputs": [],
   "source": [
    "RANDOM_SEED = 123\n",
    "torch.manual_seed(RANDOM_SEED)\n",
    "\n",
    "DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "TEXT_PORTION_SIZE = 200\n",
    "\n",
    "NUM_ITER = 20000\n",
    "LEARNING_RATE = 0.005\n",
    "EMBEDDING_DIM = 100\n",
    "HIDDEN_DIM = 100\n",
    "NUM_HIDDEN = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mQMmKUEisW4W"
   },
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4GnH64XvsV8n"
   },
   "source": [
    "Download *[A Tale of Two Cities](http://www.gutenberg.org/files/98/98-0.txt)* by Charles Dickens from the Gutenberg Project:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "id": "WZ_4jiHVnMxN",
    "outputId": "40d01460-fe20-476b-c270-1fee964bcf9f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2019-04-26 04:03:36--  http://www.gutenberg.org/files/98/98-0.txt\n",
      "Resolving www.gutenberg.org (www.gutenberg.org)... 152.19.134.47, 2610:28:3090:3000:0:bad:cafe:47\n",
      "Connecting to www.gutenberg.org (www.gutenberg.org)|152.19.134.47|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 804335 (785K) [text/plain]\n",
      "Saving to: ‘98-0.txt.11’\n",
      "\n",
      "98-0.txt.11         100%[===================>] 785.48K  1.68MB/s    in 0.5s    \n",
      "\n",
      "2019-04-26 04:03:36 (1.68 MB/s) - ‘98-0.txt.11’ saved [804335/804335]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget http://www.gutenberg.org/files/98/98-0.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "L-TBwKWPslPa"
   },
   "source": [
    "Convert all characters into ASCII characters provided by `string.printable`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "i8lk0nEnih-x",
    "outputId": "3c9c5cca-55a9-4e7e-a02b-64f8b9cea6c4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~ \\t\\n\\r\\x0b\\x0c'"
      ]
     },
     "execution_count": 4,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string.printable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "e8uNrjdtn4A8",
    "outputId": "28b78ff2-8b68-4fe8-d01e-c27c8692f3ac"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of characters in text: 776058\n"
     ]
    }
   ],
   "source": [
    "with open('./98-0.txt', 'r') as f:\n",
    "    textfile = f.read()\n",
    "\n",
    "# convert special characters\n",
    "textfile = unidecode.unidecode(textfile)\n",
    "\n",
    "# strip extra whitespaces\n",
    "textfile = re.sub(' +',' ', textfile)\n",
    "\n",
    "TEXT_LENGTH = len(textfile)\n",
    "\n",
    "print(f'Number of characters in text: {TEXT_LENGTH}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "JpEMNInXtZsb"
   },
   "source": [
    "Divide the text into smaller portions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "i7JiHR1stHNF",
    "outputId": "b99c6a1d-6906-4845-e9fe-20fad9ae9315"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " left his saw sticking in the firewood he was cutting, set it in\n",
      "motion again; the women who had left on a door-step the little pot of\n",
      "hot ashes, at which she had been trying to soften the pain in her \n"
     ]
    }
   ],
   "source": [
    "random.seed(RANDOM_SEED)\n",
    "\n",
    "def random_portion(textfile):\n",
    "    start_index = random.randint(0, TEXT_LENGTH - TEXT_PORTION_SIZE)\n",
    "    end_index = start_index + TEXT_PORTION_SIZE + 1\n",
    "    return textfile[start_index:end_index]\n",
    "\n",
    "print(random_portion(textfile))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PhZQCuBZkzgm"
   },
   "source": [
    "Define a function to convert characters into tensors of integers (type long):\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "Ga-_g7Ltk479",
    "outputId": "2dd6fcd9-01b4-4910-f57a-812d05b9a4d1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([10, 11, 12, 39, 40, 41])\n"
     ]
    }
   ],
   "source": [
    "def char_to_tensor(text):\n",
    "    lst = [string.printable.index(c) for c in text]\n",
    "    tensor = torch.tensor(lst).long()\n",
    "    return tensor\n",
    "\n",
    "print(char_to_tensor('abcDEF'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ph8OESgtmUqD"
   },
   "source": [
    "Putting it together to make a function that draws random batches for training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "U_mVm5LOl95j"
   },
   "outputs": [],
   "source": [
    "def draw_random_sample(textfile):    \n",
    "    text_long = char_to_tensor(random_portion(textfile))\n",
    "    inputs = text_long[:-1]\n",
    "    targets = text_long[1:]\n",
    "    return inputs, targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 425
    },
    "colab_type": "code",
    "id": "uY8FnmSkxGZ-",
    "outputId": "c332573b-5eb4-4367-bc4c-0a9a479aebb3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([94, 17, 18, 28, 94, 32, 18, 23, 13, 24, 32, 94, 10, 28, 94, 18, 15, 94,\n",
       "         29, 17, 10, 29, 94, 32, 14, 27, 14, 94, 27, 30, 21, 14, 13, 94, 15, 24,\n",
       "         27, 94, 15, 18, 16, 30, 27, 14, 28, 94, 29, 24, 24, 73, 94, 10, 23, 13,\n",
       "         94, 14, 31, 14, 27, 34, 29, 17, 18, 23, 16, 96, 30, 23, 13, 14, 27, 94,\n",
       "         29, 17, 14, 94, 12, 21, 24, 30, 13, 28, 94, 32, 14, 27, 14, 94, 10, 94,\n",
       "         28, 30, 22, 75, 96, 96, 63, 43, 10, 21, 21, 24, 10, 62, 63, 94, 28, 10,\n",
       "         18, 13, 94, 48, 27, 75, 94, 54, 29, 27, 34, 31, 14, 27, 75, 94, 63, 43,\n",
       "         24, 32, 94, 13, 24, 94, 34, 24, 30, 94, 13, 24, 82, 94, 44, 94, 17, 24,\n",
       "         25, 14, 94, 34, 24, 30, 94, 10, 27, 14, 94, 32, 14, 21, 21, 62, 63, 96,\n",
       "         96, 44, 29, 94, 32, 10, 28, 94, 54, 29, 27, 34, 31, 14, 27, 68, 28, 94,\n",
       "         16, 27, 10, 23, 13, 94, 25, 14, 12, 30, 21, 18, 10, 27, 18, 29, 34, 94,\n",
       "         29, 17]),\n",
       " tensor([17, 18, 28, 94, 32, 18, 23, 13, 24, 32, 94, 10, 28, 94, 18, 15, 94, 29,\n",
       "         17, 10, 29, 94, 32, 14, 27, 14, 94, 27, 30, 21, 14, 13, 94, 15, 24, 27,\n",
       "         94, 15, 18, 16, 30, 27, 14, 28, 94, 29, 24, 24, 73, 94, 10, 23, 13, 94,\n",
       "         14, 31, 14, 27, 34, 29, 17, 18, 23, 16, 96, 30, 23, 13, 14, 27, 94, 29,\n",
       "         17, 14, 94, 12, 21, 24, 30, 13, 28, 94, 32, 14, 27, 14, 94, 10, 94, 28,\n",
       "         30, 22, 75, 96, 96, 63, 43, 10, 21, 21, 24, 10, 62, 63, 94, 28, 10, 18,\n",
       "         13, 94, 48, 27, 75, 94, 54, 29, 27, 34, 31, 14, 27, 75, 94, 63, 43, 24,\n",
       "         32, 94, 13, 24, 94, 34, 24, 30, 94, 13, 24, 82, 94, 44, 94, 17, 24, 25,\n",
       "         14, 94, 34, 24, 30, 94, 10, 27, 14, 94, 32, 14, 21, 21, 62, 63, 96, 96,\n",
       "         44, 29, 94, 32, 10, 28, 94, 54, 29, 27, 34, 31, 14, 27, 68, 28, 94, 16,\n",
       "         27, 10, 23, 13, 94, 25, 14, 12, 30, 21, 18, 10, 27, 18, 29, 34, 94, 29,\n",
       "         17, 10]))"
      ]
     },
     "execution_count": 9,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_random_sample(textfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "G_grdW3pxCzz"
   },
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nQIUm5EjxFNa"
   },
   "outputs": [],
   "source": [
    "class RNN(torch.nn.Module):\n",
    "    def __init__(self, input_size, embed_size,\n",
    "                 hidden_size, output_size, num_layers):\n",
    "        super(RNN, self).__init__()\n",
    "\n",
    "        self.num_layers = num_layers\n",
    "        self.hidden_size = hidden_size\n",
    "        \n",
    "        self.embed = torch.nn.Embedding(input_size, hidden_size)\n",
    "        self.gru = torch.nn.GRU(input_size=embed_size,\n",
    "                                hidden_size=hidden_size,\n",
    "                                num_layers=num_layers)\n",
    "        self.fc = torch.nn.Linear(hidden_size, output_size)\n",
    "        self.init_hidden = torch.nn.Parameter(torch.zeros(\n",
    "                                              num_layers, 1, hidden_size))\n",
    "    \n",
    "    def forward(self, features, hidden):\n",
    "        embedded = self.embed(features.view(1, -1))\n",
    "        output, hidden = self.gru(embedded.view(1, 1, -1), hidden)\n",
    "        output = self.fc(output.view(1, -1))\n",
    "        return output, hidden\n",
    "      \n",
    "    def init_zero_state(self):\n",
    "        init_hidden = torch.zeros(self.num_layers, 1, self.hidden_size).to(DEVICE)\n",
    "        return init_hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ik3NF3faxFmZ"
   },
   "outputs": [],
   "source": [
    "torch.manual_seed(RANDOM_SEED)\n",
    "model = RNN(len(string.printable), EMBEDDING_DIM, HIDDEN_DIM, len(string.printable), NUM_HIDDEN)\n",
    "model = model.to(DEVICE)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Lv9Ny9di6VcI"
   },
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "x5XlCqm3pMj2"
   },
   "outputs": [],
   "source": [
    "def evaluate(model, prime_str='A', predict_len=100, temperature=0.8):\n",
    "    ## based on https://github.com/spro/practical-pytorch/\n",
    "    ## blob/master/char-rnn-generation/char-rnn-generation.ipynb\n",
    "\n",
    "    hidden = model.init_zero_state()\n",
    "    prime_input = char_to_tensor(prime_str)\n",
    "    predicted = prime_str\n",
    "\n",
    "    # Use priming string to \"build up\" hidden state\n",
    "    for p in range(len(prime_str) - 1):\n",
    "        _, hidden = model(prime_input[p].to(DEVICE), hidden.to(DEVICE))\n",
    "    inp = prime_input[-1]\n",
    "    \n",
    "    for p in range(predict_len):\n",
    "        output, hidden = model(inp.to(DEVICE), hidden.to(DEVICE))\n",
    "        \n",
    "        # Sample from the network as a multinomial distribution\n",
    "        output_dist = output.data.view(-1).div(temperature).exp()\n",
    "        top_i = torch.multinomial(output_dist, 1)[0]\n",
    "        \n",
    "        # Add predicted character to string and use as next input\n",
    "        predicted_char = string.printable[top_i]\n",
    "        predicted += predicted_char\n",
    "        inp = char_to_tensor(predicted_char)\n",
    "\n",
    "    return predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 3536
    },
    "colab_type": "code",
    "id": "ZNv7_wE4prjX",
    "outputId": "8ae1c36b-e4e8-4fb6-852d-ac926ad34cb8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time elapsed: 0.00 min\n",
      "Iteration 0 | Loss 4.58\n",
      "\n",
      "\n",
      "Th4izvh?=lw2ZaCV_}xEt5y.gA^+r\u000b",
      "PgO2z@<$.1KRQe/c\\\t{a5A55Dun}_*czf.o6Hmy$l\"W@\f",
      "f\u000b",
      "i{7rjKsvnEMJ\n",
      "mr`PaKygiE+VSbR#RF|SC^g^CZK,aenDc)t.O_\n",
      "D^(M]1w'^Wd_HDws\\>_2)iavp?*c-npOvoQE>i L \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 2.63 min\n",
      "Iteration 1000 | Loss 1.81\n",
      "\n",
      "\n",
      "Th Prost\n",
      "into\n",
      "he forn a wock, abrould with his lother the star a caide with the Jue turnd face. Breaknay when and and of or, street were work\n",
      "have the long is on the proseing bove wabres. Throk a mean h \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 5.29 min\n",
      "Iteration 2000 | Loss 1.72\n",
      "\n",
      "\n",
      "Ther face. And civery ire head shook the lange's was note my booked she cray. The grance for that the with Lerary swere were, and for young to-the\n",
      "wank the tanger brother whas at a for the requestone-st \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 7.91 min\n",
      "Iteration 3000 | Loss 1.73\n",
      "\n",
      "\n",
      "Thou my menal known a purntatieful a might\n",
      "Frent fargefuch by sour that reforned after as as a mists\n",
      "and the countice of the Founk\n",
      "\n",
      "\n",
      "\n",
      "\"I among him your for the you glason in?\"\n",
      "\n",
      "\"I constrance yhabuing a  \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 10.55 min\n",
      "Iteration 4000 | Loss 1.77\n",
      "\n",
      "\n",
      "The seeantelition pricomer; I have\n",
      "had the passess bestious had be patriender one up thow, such the even the line\n",
      "and that ins show was somen of his openey, but fine had a raghter?\n",
      "\n",
      "\"I! And at a sifulra \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 13.17 min\n",
      "Iteration 5000 | Loss 1.46\n",
      "\n",
      "\n",
      "The Bask tree. \"The intame!\"\n",
      "\n",
      "\"Neothing and fam and if you brow lisert, to the mouther desk to an to the Gells that immered of the\n",
      "indence an aftionation bud, undering to went remark down off work; doe! \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 15.83 min\n",
      "Iteration 6000 | Loss 1.64\n",
      "\n",
      "\n",
      "The Pross. What\n",
      "of moon, and worth her knitting, and is he see myself the was seeper on prisoner her been on him our, and\n",
      "yet in the poors; is stooness of a morned this things more, were benthell name,  \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 18.47 min\n",
      "Iteration 7000 | Loss 1.64\n",
      "\n",
      "\n",
      "The here an the ferty care it was\n",
      "of the streach. As Miss Pross Borring of her surfounds of comprelegual\n",
      "saken which his returnes, shall in Heaved the arrows\n",
      "of the retore, then for Defarge. Jark, he wa \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 21.12 min\n",
      "Iteration 8000 | Loss 1.66\n",
      "\n",
      "\n",
      "Thur and the decients than any.\n",
      "\n",
      "Monsiever such put her cite out over the cermarded and in herce then the repariey who\n",
      "grance the stalled be of the own and conversicted way of his anterom\n",
      "cold the cirse \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 23.77 min\n",
      "Iteration 9000 | Loss 1.53\n",
      "\n",
      "\n",
      "Thrat to his man that extenss of the said her and had and world at it, she had was\n",
      "as breat--how had asseet triatile of the pationed, and\n",
      "that worked he works of one and nobainly, and out of that at the \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 26.43 min\n",
      "Iteration 10000 | Loss 1.30\n",
      "\n",
      "\n",
      "Ther his moth wooten a new blood, a sile, the lactriden\n",
      "nother were noter, who had from his father to prettorers his\n",
      "fation. Then. He are is him a sloke it soits in him woired to the paper women, maning \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 29.07 min\n",
      "Iteration 11000 | Loss 1.73\n",
      "\n",
      "\n",
      "The eighs while Miss Pross was\n",
      "saying a could they last the done by, and pressed to\n",
      "the been hackeful hight in mending, and to the done into-raid to\n",
      "have little faming shall now, with the said to go of  \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 31.72 min\n",
      "Iteration 12000 | Loss 1.68\n",
      "\n",
      "\n",
      "The here. It were would done.\n",
      "\n",
      "\"It alread, I was say?\" seen in not in the culles the sunded Miss, sure to be there were the would close\n",
      "he dark see radfe taken it is instend me had done-all I spy so\n",
      "str \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 34.35 min\n",
      "Iteration 13000 | Loss 1.42\n",
      "\n",
      "\n",
      "The part, but, at had were tosen in\n",
      "it are of a proined serverently passing the fars, and the\n",
      "friended that a fiffer the knouttial backle and day, list, and from to could my\n",
      "deting; and very dark of the \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 37.00 min\n",
      "Iteration 14000 | Loss 1.78\n",
      "\n",
      "\n",
      "Thre it was a days.\n",
      "\n",
      "\"You and deperianned of the moved there way, and a socions of the proppiouse and this must a dively?\n",
      "\n",
      "\"Yest!\" (And in the care befon this there,\" asked nother, to the two in this ex \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 39.63 min\n",
      "Iteration 15000 | Loss 1.54\n",
      "\n",
      "\n",
      "Tho a man in all looking\n",
      "the mannen were trangs; he more at man and in the had believe the sick of their an\n",
      "than the man the prioned in a golderate scattered no stup, and look, all thoused shall\n",
      "law sca \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 42.28 min\n",
      "Iteration 16000 | Loss 1.52\n",
      "\n",
      "\n",
      "Thrishe forth, his have like him\n",
      "and words of it is a peeched in the eyes farge what\n",
      "it went exciect the deing and the mittions.\n",
      "\n",
      "The mounged the repalling's citines of mineurmt you not thinks,\n",
      " Charlee \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 44.93 min\n",
      "Iteration 17000 | Loss 1.58\n",
      "\n",
      "\n",
      "Thrithest the prisonened I\n",
      "staid be, short, and not morright door with with the mitting to my worthud no paid\n",
      "it.\"\n",
      "\n",
      "\"He do I as a more through a passed and go more.\n",
      "\n",
      "No and me, the far bold to fears and \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 47.56 min\n",
      "Iteration 18000 | Loss 1.49\n",
      "\n",
      "\n",
      "Tho\n",
      "you would fro in his intides rather sation and chocal went in the things, asked the have hand of the\n",
      "distened did of the cately roar chifulures. What the His of a not his have pourty\n",
      "the took this l \n",
      "\n",
      "==================================================\n",
      "Time elapsed: 50.19 min\n",
      "Iteration 19000 | Loss 1.78\n",
      "\n",
      "\n",
      "Thragges,\" said some of\n",
      "a puncher in the Gabody old, was a Fants tall to know of the complight--seat\n",
      "more inten asse interancame my any went med Courable hands in that he behing make no will never see t \n",
      "\n",
      "==================================================\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "for iteration in range(NUM_ITER):\n",
    "\n",
    "    \n",
    "    ### FORWARD AND BACK PROP\n",
    "\n",
    "    hidden = model.init_zero_state()\n",
    "    optimizer.zero_grad()\n",
    "    \n",
    "    loss = 0.\n",
    "    inputs, targets = draw_random_sample(textfile)\n",
    "    inputs, targets = inputs.to(DEVICE), targets.to(DEVICE)\n",
    "    for c in range(TEXT_PORTION_SIZE):\n",
    "        outputs, hidden = model(inputs[c], hidden)\n",
    "        loss += F.cross_entropy(outputs, targets[c].view(1))\n",
    "\n",
    "    loss /= TEXT_PORTION_SIZE\n",
    "    loss.backward()\n",
    "    \n",
    "    ### UPDATE MODEL PARAMETERS\n",
    "    optimizer.step()\n",
    "\n",
    "    ### LOGGING\n",
    "    with torch.set_grad_enabled(False):\n",
    "      if iteration % 1000 == 0:\n",
    "          print(f'Time elapsed: {(time.time() - start_time)/60:.2f} min')\n",
    "          print(f'Iteration {iteration} | Loss {loss.item():.2f}\\n\\n')\n",
    "          print(evaluate(model, 'Th', 200), '\\n')\n",
    "          print(50*'=')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mEvBe4V0ykSl"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "char-rnn.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}