GPT2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "GPT2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ayush06feb/GPT-2/blob/master/GPT2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eWwo3MhK3v2O",
        "colab_type": "text"
      },
      "source": [
        "#GPT-2 on Google Colab\n",
        "---\n",
        "This notebook allows you to easily run GPT-2 on colab servers.\n",
        "\n",
        "```\n",
        "<!> Improtant: Chck if hardware GPU accelration is enabled in current runtime.\n",
        "```\n",
        "\n",
        "<img src=\"https://i.ibb.co/F8yQm3s/Screen-Shot-2019-12-16-at-11-17-40-PM.png\" alt=\"Runtime -> Change runtime type\" border=\"0\">\n",
        "\n",
        "```\n",
        "WARNING: Samples are unfiltered and may contain offensive content.\n",
        "```\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8QmMmBs99stO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#   ██████╗ ██████╗ ████████╗   ██████╗ \n",
        "#  ██╔════╝ ██╔══██╗╚══██╔══╝   ╚════██╗\n",
        "#  ██║  ███╗██████╔╝   ██║█████╗ █████╔╝\n",
        "#  ██║   ██║██╔═══╝    ██║╚════╝██╔═══╝ \n",
        "#  ╚██████╔╝██║        ██║      ███████╗\n",
        "#   ╚═════╝ ╚═╝        ╚═╝      ╚══════╝"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G5h0LpCrlbcr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#[Important]\n",
        "# Imports tenserflow & checks version\n",
        "%tensorflow_version 1.x\n",
        "import tensorflow\n",
        "print(tensorflow.__version__)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xZ2Vl8l5i7s2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#[Not necessary]\n",
        "# Checks if you have GPU access and project system resources allocation\n",
        "\n",
        "# memory footprint support libraries/code\n",
        "!ln -sf /opt/bin/nvidia-smi /usr/bin/nvidia-smi\n",
        "!pip install gputil\n",
        "!pip install psutil\n",
        "!pip install humanize\n",
        "import psutil\n",
        "import humanize\n",
        "import os\n",
        "import GPUtil as GPU\n",
        "GPUs = GPU.getGPUs()\n",
        "# XXX: only one GPU on Colab and isn’t guaranteed\n",
        "gpu = GPUs[0]\n",
        "def printm():\n",
        " process = psutil.Process(os.getpid())\n",
        " print(\"Gen RAM Free: \" + humanize.naturalsize( psutil.virtual_memory().available ), \" | Proc size: \" + humanize.naturalsize( process.memory_info().rss))\n",
        " print(\"GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB\".format(gpu.memoryFree, gpu.memoryUsed, gpu.memoryUtil*100, gpu.memoryTotal))\n",
        "printm()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P7ZyH5cuvXLe",
        "colab_type": "text"
      },
      "source": [
        "#### Model size:\n",
        "Different model sizes are available of GPT2-\n",
        "\n",
        "<table>\n",
        "  <tr>\n",
        "  <th>Model Size</th>\n",
        "  <th>Info</th>\n",
        "  </tr>\n",
        "  <tr>\n",
        "  <td>117M</td>\n",
        "  <td>Smallest model. Fastest but not accurate</td>\n",
        "  </tr>\n",
        "  <tr>\n",
        "  <td>124M</td>\n",
        "  <td>Bigger than 117M, but still not very accurate</td>\n",
        "  </tr>\n",
        "  <tr>\n",
        "  <td>355M</td>\n",
        "  <td>Can be used on low-end machines after training</td>\n",
        "  </tr>\n",
        "  <tr>\n",
        "  <td>774M</td>\n",
        "  <td>Best balance between speed and quality</td>\n",
        "  </tr>\n",
        "  <tr>\n",
        "  <td>1558M</td>\n",
        "  <td>Biggest 1.5B model. Takes longer to run</td>\n",
        "  </tr>\n",
        " </table>\n",
        "\n",
        "=> Change `model_name` to change model size.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jLRA3PTsSclp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Select GPT-2 model: 117M, 124M, 355M, 774M, 1558M\n",
        "#####################\n",
        "model_name = '774M'     # '117M' - smallest model. Set '1558M' for biggest 1.5B model\n",
        "#####################\n",
        "!git clone https://github.com/openai/gpt-2\n",
        "\n",
        "%cd gpt-2\n",
        "\n",
        "!pip3 install -r requirements.txt\n",
        "\n",
        "# Download GPT-2 (selected model)\n",
        "!python3 download_model.py $model_name\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "va7ELqZuS0tR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Generate samples by user input. Wait for string \"Model prompt >>>\", enter you text (begin phrase for network) and press Enter\n",
        "\n",
        "!python3 src/interactive_conditional_samples.py --model_name"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8XlbjCjHS24j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Generate samples to file samples.txt. To stop press STOP button at this cell. To download result samples.txt see last cell\n",
        "\n",
        "!python3 src/generate_unconditional_samples.py --model_name=$model_name | tee samples.txt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0bXQ5wRwS_bB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Generate samples with parameters: top_k and temperature. Result saved to samples.txt\n",
        "\n",
        "!python3 src/generate_unconditional_samples.py --model_name=$model_name --top_k 40 --temperature 0.7 | tee samples.txt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8muZAgW9TQgv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Download file samples.txt with generated text (unconditional mode only)\n",
        "\n",
        "from google.colab import files\n",
        "\n",
        "files.download('samples.txt')\n"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}