nlp/Basic_NLP_with_PyTorch_Text.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Basic NLP with PyTorch Text.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "osQQp4NPvVJC",
        "colab_type": "text"
      },
      "source": [
        "# Building a Language Model dataset using PyTorch Text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vYGHDjNVn1-b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "import torchtext as tt\n",
        "from collections import Counter\n",
        "import functools\n",
        "import operator"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wwNu_yqIu8o2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "spacy_en = spacy.load('en')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lYZ8O-8svBi7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "seq_len  =  15#@param"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2j0JpTX4PG_q",
        "colab_type": "text"
      },
      "source": [
        "## Text processing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "34MzMHr8vNSe",
        "colab_type": "text"
      },
      "source": [
        "Create a tokenizer function using Spacy.\n",
        "> By default, PyTorch Text uses a whitespace tokenization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XrlUyN26vFvV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def spacy_tokenizer(text):\n",
        "  return [tok.text for tok in spacy_en.tokenizer(text)]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8bwAD50qvROO",
        "colab_type": "text"
      },
      "source": [
        "Create a text processing pipeline using PyTorch Text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J_hsNvhhvNz9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "TEXT = tt.data.Field(\n",
        "  tokenize    = spacy_tokenizer,\n",
        "  lower       = True,\n",
        "  batch_first = True,\n",
        "  init_token  = '<bos>',\n",
        "  eos_token   = '<eos>',\n",
        "  fix_length  = seq_len\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j5eCWo3axw3t",
        "colab_type": "text"
      },
      "source": [
        "Text Preprocessing:\n",
        "* Lowercasting text: as flag `lower` in TEXT is set to `true`\n",
        "* Tokenizing text: as a tokenization function was provided"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4d65R8PUvPz2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "minibatch = [ 'The Brown Fox Jumped Over The Lazy Dog' ]\n",
        "minibatch = list(map(TEXT.preprocess, minibatch))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sE8speNV076_",
        "colab_type": "text"
      },
      "source": [
        "Padding text with the `<pad>` token so that the sequence length is  matched"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3FDON98V06Au",
        "colab_type": "code",
        "outputId": "91499cd3-0fd2-40d2-b629-08aaf83c426e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "minibatch = TEXT.pad(minibatch)\n",
        "print(minibatch)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[['<bos>', 'the', 'brown', 'fox', 'jumped', 'over', 'the', 'lazy', 'dog', '<eos>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wgy0l2w40cUd",
        "colab_type": "text"
      },
      "source": [
        "Manually build a vocab"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vDjqK7hQ00rj",
        "colab_type": "code",
        "outputId": "4e5a271f-c208-4c26-df5f-f1543b6a2a96",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 185
        }
      },
      "source": [
        "tokens = functools.reduce(operator.concat, minibatch)\n",
        "counter = Counter(tokens)\n",
        "counter"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Counter({'<bos>': 1,\n",
              "         '<eos>': 1,\n",
              "         '<pad>': 5,\n",
              "         'brown': 1,\n",
              "         'dog': 1,\n",
              "         'fox': 1,\n",
              "         'jumped': 1,\n",
              "         'lazy': 1,\n",
              "         'over': 1,\n",
              "         'the': 2})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ekglykoA0neX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vocab = TEXT.vocab_cls(counter)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eEvH7pNt1pzO",
        "colab_type": "code",
        "outputId": "428f2be0-fb43-491a-8926-1ef16d3fc34a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(vocab.itos)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['<pad>', 'the', '<bos>', '<eos>', 'brown', 'dog', 'fox', 'jumped', 'lazy', 'over']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VFpnyRMs3U_i",
        "colab_type": "code",
        "outputId": "223f512f-d896-43ad-e287-3091655b3ddc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "print(vocab.stoi)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "defaultdict(<function _default_unk_index at 0x7ff038fb6a60>, {'<pad>': 0, 'the': 1, '<bos>': 2, '<eos>': 3, 'brown': 4, 'dog': 5, 'fox': 6, 'jumped': 7, 'lazy': 8, 'over': 9})\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "isNjPTyM_QmJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "TEXT.vocab = vocab"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9E0sIatG_TrM",
        "colab_type": "code",
        "outputId": "c5b1689d-3a31-41ac-8f5c-f87d68e48b80",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "TEXT.numericalize(minibatch)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[2, 1, 4, 6, 7, 9, 1, 8, 5, 3, 0, 0, 0, 0, 0]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cs-_mLMR4Tfn",
        "colab_type": "text"
      },
      "source": [
        "Automatically construct a vocab"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V1MU1zyG3ZRD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "TEXT.build_vocab(minibatch)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4WPOdbAE356N",
        "colab_type": "code",
        "outputId": "1bbc647e-1268-40c4-aa7b-15c9ed2eb204",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(TEXT.vocab.itos)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['<unk>', '<pad>', '<bos>', '<eos>', 'the', 'brown', 'dog', 'fox', 'jumped', 'lazy', 'over']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j9Xk4NCZ4YaS",
        "colab_type": "code",
        "outputId": "7b64c882-f452-4c81-ffa4-44c092188515",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "print(TEXT.vocab.stoi)"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "defaultdict(<function _default_unk_index at 0x7ff038fb6a60>, {'<unk>': 0, '<pad>': 1, '<bos>': 2, '<eos>': 3, 'the': 4, 'brown': 5, 'dog': 6, 'fox': 7, 'jumped': 8, 'lazy': 9, 'over': 10})\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EWgYj1IVzeKG",
        "colab_type": "code",
        "outputId": "c5d980af-73d3-4dba-8c79-5109bb93139d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "TEXT.numericalize(minibatch)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 2,  4,  5,  7,  8, 10,  4,  9,  6,  3,  1,  1,  1,  1,  1]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cx0Pan4yPM-I",
        "colab_type": "text"
      },
      "source": [
        "## Data Loader"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "15rxnKPhPo49",
        "colab_type": "text"
      },
      "source": [
        "Build a dataset given a training and validation text files, and using the previously built text processing pipeline."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IYGeVuhcyhtD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_ds, valid_ds = tt.data.TabularDataset.splits(\n",
        "    path=PATH,\n",
        "    train='train.csv',\n",
        "    validation='valid.csv',\n",
        "    format='csv',\n",
        "    fields=[('text', TEXT)]\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TlIacEKoUbLW",
        "colab_type": "text"
      },
      "source": [
        "### Data Loader for Language Modeling"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nWWKhMMPQkvn",
        "colab_type": "text"
      },
      "source": [
        "This dataset can be used to build an iterator that produces data for multiple NLP Tasks. For instance, to build the samples to use for Language Modeling using [torchtext.data.BPTTIterator](https://torchtext.readthedocs.io/en/latest/data.html#bpttiterator)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jGUtsdkLPT9W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def dataset2example(dataset, field):\n",
        "    examples = list(map(lambda example: ['<bos>']+ example.text + ['<eos>'], dataset.examples))\n",
        "    examples = [item for example in examples for item in example]\n",
        "    example = tt.data.Example()\n",
        "    setattr(example, 'text', examples)\n",
        "    return tt.data.Dataset([example], fields={'text': field})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N1rKtpBCTsOn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_example = dataset2example(train_ds, TEXT)\n",
        "valid_example = dataset2example(valid_ds, TEXT)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7xQQnNvjTssf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_iter, valid_iter = tt.data.BPTTIterator.splits((train_example, valid_example), batch_size=batch_size, bptt_len=30)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IRQdakn3VhxE",
        "colab_type": "text"
      },
      "source": [
        "The resulting `train_iter` and `valid_iter` are iterators over batches of samples that can be used in a training loop."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OZrlrvaOViXX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}