notebooks/text_generator.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": true
      },
      "source": [
        "#  English sequence generator \n",
        "\n",
        "**Creating an English language sequence generator capable of building semi-coherent English sentences from scratch by building them up character-by-character**\n",
        "\n",
        "**Natural Language Processing**\n",
        "\n",
        "Dataset:  Complete version of Sir Arthur Conan Doyle's classic book The Adventures of Sherlock Holmes\n",
        "\n",
        "Based on [RNN project: text generation](https://github.com/udacity/aind2-rnn) of the [Udacity's Artificial Intelligence  Nanodegree](https://www.udacity.com/course/artificial-intelligence-nanodegree--nd889)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "%matplotlib inline\n",
        "\n",
        "import os, sys\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import ds_boost\n",
        "import keras\n",
        "\n",
        "log = ds_boost.logger.init(level=\"DEBUG\", save_log=False)\n",
        "\n",
        "ds_boost.set_parent_execution_path()\n",
        "ds_boost.info_system()\n",
        "ds_boost.reproducible(seed=9)\n",
        "sns.set()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Load and Process the data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "text = open(\"data/holmes.txt\").read().lower()\n",
        "print(\"Total characters: {}\".format(len(text)))\n",
        "text[:300]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Preprocess the data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "text = text[1302:]  # remove title, author page, and table of contents\n",
        "text = text.replace(\"\\n\", \" \")\n",
        "text = text.replace(\"\\r\", \" \")\n",
        "\n",
        "unique_characters = set(list(text))\n",
        "print(unique_characters)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# remove non-english characters\n",
        "import re\n",
        "\n",
        "text = re.sub(\"[$%&'()*@/\u00e0\u00e2\u00e8\u00e90123456789-]\", \" \", text)\n",
        "text = text.replace('\"', \" \")\n",
        "text = text.replace(\"  \", \" \")  # shorten any extra dead space created above\n",
        "text[:300]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "chars = sorted(list(set(text)))\n",
        "num_chars = len(chars)\n",
        "print(\"Total characters: {}\".format(len(text)))\n",
        "print(\"Unique characters: {}\".format(num_chars))\n",
        "print(chars)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Split data into input/output pairs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Transforms the input text and window-size into a set of input/output pairs\n",
        "#  for use with the RNN \"\"\"\n",
        "\n",
        "window_size = 100\n",
        "step_size = 5\n",
        "\n",
        "input_pairs = []\n",
        "output_pairs = []\n",
        "\n",
        "for i in range(0, len(text) - window_size, step_size):\n",
        "    input_pairs.append(text[i : i + window_size])\n",
        "    output_pairs.append(text[i + window_size])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### One-hot encoding characters"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "chars_to_indices = dict((c, i) for i, c in enumerate(chars))\n",
        "indices_to_chars = dict((i, c) for i, c in enumerate(chars))\n",
        "\n",
        "# create variables for one-hot encoded input/output\n",
        "X = np.zeros((len(input_pairs), window_size, num_chars), dtype=bool)\n",
        "y = np.zeros((len(input_pairs), num_chars), dtype=bool)\n",
        "\n",
        "# transform character-based input_pairs/output_pairs into equivalent numerical versions\n",
        "for i, sentence in enumerate(input_pairs):\n",
        "    for t, char in enumerate(sentence):\n",
        "        X[i, t, chars_to_indices[char]] = 1\n",
        "    y[i, chars_to_indices[output_pairs[i]]] = 1"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Recurrent Neural Network Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Activation, LSTM\n",
        "\n",
        "model = Sequential()\n",
        "model.add(LSTM(200, input_shape=(window_size, num_chars)))\n",
        "model.add(Dense(num_chars, activation=None))\n",
        "model.add(Dense(num_chars, activation=\"softmax\"))\n",
        "model.summary()\n",
        "\n",
        "optimizer = keras.optimizers.RMSprop(learning_rate=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\n",
        "\n",
        "model.compile(loss=\"categorical_crossentropy\", optimizer=optimizer)\n",
        "\n",
        "# train the model\n",
        "print(\"Training ... (this may take a while)\")\n",
        "%time history = model.fit(X, y, batch_size=512, epochs=100,verbose=0)\n",
        "ds_boost.show_training(history)\n",
        "\n",
        "model_path = os.path.join(\"models\", \"text_generator.h5\")\n",
        "model.save(model_path)\n",
        "print(\"\\nModel saved at\", model_path)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Make predictions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "model = keras.models.load_model(model_path)\n",
        "print(\"Model loaded:\", model_path)\n",
        "\n",
        "\n",
        "def predict_next_chars(model, input_chars, num_to_predict):\n",
        "    \"\"\"predict a number of future characters\"\"\"\n",
        "\n",
        "    predicted_chars = \"\"\n",
        "    for i in range(num_to_predict):\n",
        "        x_test = np.zeros((1, window_size, len(chars)))\n",
        "        for t, char in enumerate(input_chars):\n",
        "            x_test[0, t, chars_to_indices[char]] = 1.0\n",
        "\n",
        "        test_predict = model.predict(x_test, verbose=0)[0]\n",
        "\n",
        "        # translate numerical prediction back to characters\n",
        "        r = np.argmax(test_predict)\n",
        "        d = indices_to_chars[r]\n",
        "\n",
        "        # update predicted_chars and input\n",
        "        predicted_chars += d\n",
        "        input_chars += d\n",
        "        input_chars = input_chars[1:]\n",
        "    return predicted_chars\n",
        "\n",
        "\n",
        "for s in range(0, 500, 100):\n",
        "    start_index = s\n",
        "    input_chars = text[start_index : start_index + window_size]\n",
        "    predict_input = predict_next_chars(model, input_chars, num_to_predict=100)\n",
        "\n",
        "    print(\"------------------\")\n",
        "    input_line = \"input chars = \" + \"\\n\" + input_chars + '\"' + \"\\n\"\n",
        "    print(input_line)\n",
        "\n",
        "    line = \"predicted chars = \" + \"\\n\" + predict_input + '\"' + \"\\n\"\n",
        "    print(line)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "ds-keras",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.4"
    },
    "vscode": {
      "interpreter": {
        "hash": "366a4202e041eebdd8a4edd8a0024926c2e45e658d7ba92f3c9b74c9b0ce97a9"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}