text_generation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.layers import LSTM, Dense, Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.callbacks import ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SEQ_LENGTH = 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we've imported everything we need form Keras, we're all set to go!\n",
    "\n",
    "First, we load our data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What np_utils.to_categorical does"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1  2  0  4  3  7 10]\n",
      "[[ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  1.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 1.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  1.]]\n"
     ]
    }
   ],
   "source": [
    "test_x = np.array([1, 2, 0, 4, 3, 7, 10])\n",
    "\n",
    "# one hot encoding\n",
    "test_y = np_utils.to_categorical(test_x)\n",
    "print(test_x)\n",
    "print(test_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This functions returns an array of sequences from the input text file and the corresponding output for each sequence encoded as a one-hot vector."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we add a function to create our LSTM."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Using keras functional model\n",
    "def create_functional_model(n_layers, input_shape, hidden_dim, n_out, **kwargs):\n",
    "    drop        = kwargs.get('drop_rate', 0.2)\n",
    "    activ       = kwargs.get('activation', 'softmax')\n",
    "    mode        = kwargs.get('mode', 'train')\n",
    "    hidden_dim  = int(hidden_dim)\n",
    "\n",
    "    inputs      = Input(shape = (input_shape[1], input_shape[2]))\n",
    "    model       = LSTM(hidden_dim, return_sequences = True)(inputs)\n",
    "    model       = Dropout(drop)(model)\n",
    "    model       = Dense(n_out)(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Using keras sequential model\n",
    "def create_model(n_layers, input_shape, hidden_dim, n_out, **kwargs):\n",
    "    drop        = kwargs.get('drop_rate', 0.2)\n",
    "    activ       = kwargs.get('activation', 'softmax')\n",
    "    mode        = kwargs.get('mode', 'train')\n",
    "    hidden_dim  = int(hidden_dim)\n",
    "    model       = Sequential()\n",
    "    flag        = True \n",
    "\n",
    "    if n_layers == 1:   \n",
    "        model.add( LSTM(hidden_dim, input_shape = (input_shape[1], input_shape[2])) )\n",
    "        if mode == 'train':\n",
    "            model.add( Dropout(drop) )\n",
    "\n",
    "    else:\n",
    "        model.add( LSTM(hidden_dim, input_shape = (input_shape[1], input_shape[2]), return_sequences = True) )\n",
    "        if mode == 'train':\n",
    "            model.add( Dropout(drop) )\n",
    "        for i in range(n_layers - 2):\n",
    "            model.add( LSTM(hidden_dim, return_sequences = True) )\n",
    "            if mode == 'train':\n",
    "                model.add( Dropout(drop) )\n",
    "        model.add( LSTM(hidden_dim) )\n",
    "\n",
    "    model.add( Dense(n_out, activation = activ) )\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Now we train our model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train(model, X, Y, n_epochs, b_size, vocab_size, **kwargs):    \n",
    "    loss            = kwargs.get('loss', 'categorical_crossentropy')\n",
    "    opt             = kwargs.get('optimizer', 'adam')\n",
    "    \n",
    "    model.compile(loss = loss, optimizer = opt)\n",
    "\n",
    "    filepath        = \"Weights/weights-improvement-{epoch:02d}-{loss:.4f}.hdf5\"\n",
    "    checkpoint      = ModelCheckpoint(filepath, monitor = 'loss', verbose = 1, save_best_only = True, mode = 'min')\n",
    "    callbacks_list  = [checkpoint]\n",
    "    X               = X / float(vocab_size)\n",
    "    model.fit(X, Y, epochs = n_epochs, batch_size = b_size, callbacks = callbacks_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The fit function will run the input batchwase n_epochs number of times and it will save the weights to a file whenever there is an improvement. This is taken care of through the callback. <br><br>\n",
    "After the training is done or once you find a loss that you are happy with, you can test how well the model generates text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generate_text(model, X, filename, ix_to_char, vocab_size):\n",
    "    \n",
    "    # Load the weights from the epoch with the least loss\n",
    "    model.load_weights(filename)\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer = 'adam')\n",
    "\n",
    "    start   = np.random.randint(0, len(X) - 1)\n",
    "    pattern = np.ravel(X[start]).tolist()\n",
    "\n",
    "    # We seed the model with a random sequence of 100 so it can start predicting\n",
    "    print (\"Seed:\")\n",
    "    print (\"\\\"\", ''.join([ix_to_char[value] for value in pattern]), \"\\\"\")\n",
    "    output = []\n",
    "    for i in range(250):\n",
    "        x           = np.reshape(pattern, (1, len(pattern), 1))\n",
    "        x           = x / float(vocab_size)\n",
    "        prediction  = model.predict(x, verbose = 0)\n",
    "        index       = np.argmax(prediction)\n",
    "        result      = index\n",
    "        output.append(result)\n",
    "        pattern.append(index)\n",
    "        pattern = pattern[1 : len(pattern)]\n",
    "\n",
    "    print(\"Predictions\")\n",
    "    print (\"\\\"\", ''.join([ix_to_char[value] for value in output]), \"\\\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we're ready to either train or test our model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "List of unique characters : \n",
      " ['\\n', ' ', '!', '&', \"'\", '(', ')', ',', '-', '.', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '?', '[', ']', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']\n",
      "Number of unique characters : \n",
      " 51\n",
      "Character to integer mapping : \n",
      " {'\\n': 0, ' ': 1, '!': 2, '&': 3, \"'\": 4, '(': 5, ')': 6, ',': 7, '-': 8, '.': 9, '0': 10, '1': 11, '2': 12, '3': 13, '4': 14, '5': 15, '6': 16, '7': 17, '8': 18, '9': 19, ':': 20, ';': 21, '?': 22, '[': 23, ']': 24, 'a': 25, 'b': 26, 'c': 27, 'd': 28, 'e': 29, 'f': 30, 'g': 31, 'h': 32, 'i': 33, 'j': 34, 'k': 35, 'l': 36, 'm': 37, 'n': 38, 'o': 39, 'p': 40, 'q': 41, 'r': 42, 's': 43, 't': 44, 'u': 45, 'v': 46, 'w': 47, 'x': 48, 'y': 49, 'z': 50}\n"
     ]
    }
   ],
   "source": [
    "filename    = 'data/game_of_thrones.txt'\n",
    "data        = open(filename).read()\n",
    "data        = data.lower()\n",
    "# Find all the unique characters\n",
    "chars       = sorted(list(set(data)))\n",
    "char_to_int = dict((c, i) for i, c in enumerate(chars))\n",
    "ix_to_char  = dict((i, c) for i, c in enumerate(chars))\n",
    "vocab_size  = len(chars)\n",
    "\n",
    "print(\"List of unique characters : \\n\", chars)\n",
    "\n",
    "print(\"Number of unique characters : \\n\", vocab_size)\n",
    "\n",
    "print(\"Character to integer mapping : \\n\", char_to_int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sequences in data set : \n",
      " 1605865\n",
      "[25, 1, 43, 39, 38, 31, 1, 39, 30, 1, 33, 27, 29, 1, 25, 38, 28, 1, 30, 33, 42, 29, 0, 0, 25, 1, 31, 25, 37, 29, 1, 39, 30, 1, 44, 32, 42, 39, 38, 29, 43, 0, 0, 40, 42, 39, 36, 39, 31, 45, 29, 0, 0, 47, 29, 1, 43, 32, 39, 45, 36, 28, 1, 43, 44, 25, 42, 44, 1, 26, 25, 27, 35, 7, 1, 31, 25, 42, 29, 28, 1, 45, 42, 31, 29, 28, 1, 25, 43, 1, 44, 32, 29, 1, 47, 39, 39, 28, 43, 1]\n",
      "[1, 43, 39, 38, 31, 1, 39, 30, 1, 33, 27, 29, 1, 25, 38, 28, 1, 30, 33, 42, 29, 0, 0, 25, 1, 31, 25, 37, 29, 1, 39, 30, 1, 44, 32, 42, 39, 38, 29, 43, 0, 0, 40, 42, 39, 36, 39, 31, 45, 29, 0, 0, 47, 29, 1, 43, 32, 39, 45, 36, 28, 1, 43, 44, 25, 42, 44, 1, 26, 25, 27, 35, 7, 1, 31, 25, 42, 29, 28, 1, 45, 42, 31, 29, 28, 1, 25, 43, 1, 44, 32, 29, 1, 47, 39, 39, 28, 43, 1, 26]\n"
     ]
    }
   ],
   "source": [
    "list_X      = []\n",
    "list_Y      = []\n",
    "\n",
    "# Python append is faster than numpy append. Try it!\n",
    "for i in range(0, len(data) - SEQ_LENGTH, 1):\n",
    "    seq_in  = data[i : i + SEQ_LENGTH]\n",
    "    seq_out = data[i + SEQ_LENGTH]\n",
    "    list_X.append([char_to_int[char] for char in seq_in])\n",
    "    list_Y.append(char_to_int[seq_out])\n",
    "\n",
    "n_patterns  = len(list_X)\n",
    "print(\"Number of sequences in data set : \\n\", n_patterns)\n",
    "print(list_X[0])\n",
    "print(list_X[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[25]\n",
      " [ 1]\n",
      " [43]\n",
      " [39]\n",
      " [38]\n",
      " [31]\n",
      " [ 1]\n",
      " [39]\n",
      " [30]\n",
      " [ 1]\n",
      " [33]\n",
      " [27]\n",
      " [29]\n",
      " [ 1]\n",
      " [25]\n",
      " [38]\n",
      " [28]\n",
      " [ 1]\n",
      " [30]\n",
      " [33]\n",
      " [42]\n",
      " [29]\n",
      " [ 0]\n",
      " [ 0]\n",
      " [25]\n",
      " [ 1]\n",
      " [31]\n",
      " [25]\n",
      " [37]\n",
      " [29]\n",
      " [ 1]\n",
      " [39]\n",
      " [30]\n",
      " [ 1]\n",
      " [44]\n",
      " [32]\n",
      " [42]\n",
      " [39]\n",
      " [38]\n",
      " [29]\n",
      " [43]\n",
      " [ 0]\n",
      " [ 0]\n",
      " [40]\n",
      " [42]\n",
      " [39]\n",
      " [36]\n",
      " [39]\n",
      " [31]\n",
      " [45]\n",
      " [29]\n",
      " [ 0]\n",
      " [ 0]\n",
      " [47]\n",
      " [29]\n",
      " [ 1]\n",
      " [43]\n",
      " [32]\n",
      " [39]\n",
      " [45]\n",
      " [36]\n",
      " [28]\n",
      " [ 1]\n",
      " [43]\n",
      " [44]\n",
      " [25]\n",
      " [42]\n",
      " [44]\n",
      " [ 1]\n",
      " [26]\n",
      " [25]\n",
      " [27]\n",
      " [35]\n",
      " [ 7]\n",
      " [ 1]\n",
      " [31]\n",
      " [25]\n",
      " [42]\n",
      " [29]\n",
      " [28]\n",
      " [ 1]\n",
      " [45]\n",
      " [42]\n",
      " [31]\n",
      " [29]\n",
      " [28]\n",
      " [ 1]\n",
      " [25]\n",
      " [43]\n",
      " [ 1]\n",
      " [44]\n",
      " [32]\n",
      " [29]\n",
      " [ 1]\n",
      " [47]\n",
      " [39]\n",
      " [39]\n",
      " [28]\n",
      " [43]\n",
      " [ 1]]\n",
      "[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n"
     ]
    }
   ],
   "source": [
    "X           = np.reshape(list_X, (n_patterns, SEQ_LENGTH, 1)) # (n, 100, 1)\n",
    "# Encode output as one-hot vector\n",
    "Y           = np_utils.to_categorical(list_Y)\n",
    "\n",
    "print(X[0])\n",
    "print(Y[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of input data  (1605865, 100, 1) \n",
      "Shape of output data  (1605865, 51)\n"
     ]
    }
   ],
   "source": [
    "print(\"Shape of input data \", X.shape, \"\\nShape of output data \", Y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model   = create_model(1, X.shape, 256, Y.shape[1], mode = 'train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      " 512/1024 [==============>...............] - ETA: 6s - loss: 3.9281Epoch 00000: loss improved from inf to 3.91150, saving model to Weights/weights-improvement-00-3.9115.hdf5\n",
      "1024/1024 [==============================] - 12s - loss: 3.9115    \n",
      "Epoch 2/2\n",
      " 512/1024 [==============>...............] - ETA: 5s - loss: 3.8554Epoch 00001: loss improved from 3.91150 to 3.83877, saving model to Weights/weights-improvement-01-3.8388.hdf5\n",
      "1024/1024 [==============================] - 10s - loss: 3.8388    \n"
     ]
    }
   ],
   "source": [
    "train(model, X[:1024], Y[:1024], 2, 512, vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seed:\n",
      "\" fully.  heartsbane. lord randyll let me hold it a few times, but it always scared me. it was valyria \"\n",
      "Predictions\n",
      "\" n soaek of the tooes of the soon of the tooeses and the soon of the words and the soon of the words and the soon of the words and the soon of the words and the soon of the words and the soon of the words and the soon of the words and the soon of the  \"\n"
     ]
    }
   ],
   "source": [
    "generate_text(model, X, \"Weights/weights-improvement-36-1.7693.hdf5\", ix_to_char, vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seed:\n",
      "\" ible on the ramparts and at the gates.\n",
      "\n",
      "janos slynt met them at the door to the throne room, armored \"\n",
      "Predictions\n",
      "\"  the soon to the soon of the soot of the soot wht was a shalo oo the soow of the soot of the soot when they were so then and the soon of the soall sanears of the soadl of the soot of the soot when they were so then and the soon of the soall sanears o \"\n"
     ]
    }
   ],
   "source": [
    "generate_text(model, X, \"Weights/weights-improvement-56-1.7114.hdf5\", ix_to_char, vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}