mingpt_play_char_eksi.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    },
    "colab": {
      "name": "play_char_eksi.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kfi7pGRx5wdL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# All the credits goes to Andrei Karpathy. This is just a small adaptaion for Turkish Eksi Sozluk entries. \n",
        "# Please check out his repo - https://github.com/karpathy/minGPT"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EjOAxFpcRU3y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Hasan Ocak'ın hazırladığı eksi verisini indirmeniz lazım\n",
        "#https://www.kaggle.com/ocakhsn/eksi-sozluk-entries/version/3"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wD_CGVQp8crU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "15c94405-e940-4152-be9f-48974354313c"
      },
      "source": [
        "!unzip *.zip "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  803190_1394039_bundle_archive.zip\n",
            "  inflating: real.csv                \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n2rC4Q2H8uzb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KPqLOSPR9G5n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pandas.read_csv('real.csv', skipinitialspace=True, usecols=[3])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VGd9XUMD9Zz_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = df.iloc[:,0].str.replace(\"\\n\",\"\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Ypi15fqsRGG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "bc084af1-41e9-41d1-fb7d-6f7a1a168488"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    hala kimsenin entry girmediği başlık. daha ne ...\n",
              "1                           şu an 1$=6,87 tl olan kur.\n",
              "2                     (bkz:yatırım tavsiyesi değildir)\n",
              "3    sabah 10-11 gibi bi atak yapmasını beklediğim ...\n",
              "4    berat albayrak yumruk atmadığı için 7 tl'ye do...\n",
              "Name: text, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U7ZyiiqI9wmR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df.to_csv('eksi.txt', header=None, index=None, mode='a')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iRpMq4kk_c8E",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "0f4cacbc-e3f0-4748-8b67-4ca671e41394"
      },
      "source": [
        "!git clone https://github.com/karpathy/minGPT.git"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'minGPT'...\n",
            "remote: Enumerating objects: 21, done.\u001b[K\n",
            "remote: Total 21 (delta 0), reused 0 (delta 0), pack-reused 21\u001b[K\n",
            "Unpacking objects: 100% (21/21), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HbGkFCpZ_u-i",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e9b42fd2-b42c-44c4-8a04-36b05edd27b8"
      },
      "source": [
        "%cd minGPT/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/minGPT\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KhNdmtq88DrI",
        "colab_type": "text"
      },
      "source": [
        "## Train a character-level GPT on some text data\n",
        "\n",
        "The inputs here are simple text files, which we chop up to individual characters and then train GPT on. So you could say this is a char-transformer instead of a char-rnn. Doesn't quite roll off the tongue as well. In this example we will feed it some shakespear, which we'll get it to predict character-level."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "njGbv3sv8DrK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# set up logging\n",
        "import logging\n",
        "logging.basicConfig(\n",
        "        format=\"%(asctime)s - %(levelname)s - %(name)s -   %(message)s\",\n",
        "        datefmt=\"%m/%d/%Y %H:%M:%S\",\n",
        "        level=logging.INFO,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_ExOgiou8DrR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# make deterministic\n",
        "from mingpt.utils import set_seed\n",
        "set_seed(42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J5xYOU6U8DrV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.nn import functional as F"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sML8mgXV8Dra",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import math\n",
        "from torch.utils.data import Dataset\n",
        "\n",
        "class CharDataset(Dataset):\n",
        "\n",
        "    def __init__(self, data, block_size):\n",
        "        chars = list(set(data))\n",
        "        data_size, vocab_size = len(data), len(chars)\n",
        "        print('data has %d characters, %d unique.' % (data_size, vocab_size))\n",
        "        \n",
        "        self.stoi = { ch:i for i,ch in enumerate(chars) }\n",
        "        self.itos = { i:ch for i,ch in enumerate(chars) }\n",
        "        self.block_size = block_size\n",
        "        self.vocab_size = vocab_size\n",
        "        self.data = data\n",
        "    \n",
        "    def __len__(self):\n",
        "        return math.ceil(len(self.data) / (self.block_size + 1))\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        # we're actually going to \"cheat\" and pick a spot in the dataset at random\n",
        "        i = np.random.randint(0, len(self.data) - (self.block_size + 1))\n",
        "        chunk = self.data[i:i+self.block_size+1]\n",
        "        dix = [self.stoi[s] for s in chunk]\n",
        "        x = torch.tensor(dix[:-1], dtype=torch.long)\n",
        "        y = torch.tensor(dix[1:], dtype=torch.long)\n",
        "        return x, y\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Weizyc9t8Drf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "block_size = 128 # spatial extent of the model for its context"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jlagGiYC8Drj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "9a80d4de-ece5-43b3-e57e-61ab54357fee"
      },
      "source": [
        "# you can download this file at https://github.com/karpathy/char-rnn/blob/master/data/tinyshakespeare/input.txt\n",
        "text = open('/content/eksi.txt', 'r').read().rstrip() # don't worry we won't run out of file handles\n",
        "train_dataset = CharDataset(text, block_size) # one line of poem is roughly 50 characters\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "data has 2641167 characters, 94 unique.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gRIFfT2O8Drp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d8f88498-fb46-4f09-c301-4f9f7767e7f1"
      },
      "source": [
        "from mingpt.model import GPT, GPTConfig\n",
        "mconf = GPTConfig(train_dataset.vocab_size, train_dataset.block_size,\n",
        "                  n_layer=32, n_head=32, n_embd=512)\n",
        "model = GPT(mconf)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "08/21/2020 06:10:11 - INFO - mingpt.model -   number of parameters: 1.010391e+08\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hjWoxIpq8Dru",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "ef52926c-ccbf-458f-a24a-d84f1f2439df"
      },
      "source": [
        "from mingpt.trainer import Trainer, TrainerConfig\n",
        "\n",
        "# initialize a trainer instance and kick off training\n",
        "tconf = TrainerConfig(max_epochs=50, batch_size=32, learning_rate=6e-4,\n",
        "                      lr_decay=True, warmup_tokens=512*20, final_tokens=200*len(train_dataset)*block_size,\n",
        "                      num_workers=4)\n",
        "trainer = Trainer(model, train_dataset, None, tconf)\n",
        "trainer.train()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "epoch 1 iter 639: train loss 2.20949. lr 5.999633e-04: 100%|██████████| 640/640 [07:33<00:00,  1.41it/s]\n",
            "epoch 2 iter 639: train loss 1.87224. lr 5.998525e-04: 100%|██████████| 640/640 [07:33<00:00,  1.41it/s]\n",
            "epoch 3 iter 639: train loss 1.68608. lr 5.996678e-04: 100%|██████████| 640/640 [07:34<00:00,  1.41it/s]\n",
            "epoch 4 iter 639: train loss 1.56334. lr 5.994092e-04: 100%|██████████| 640/640 [07:35<00:00,  1.41it/s]\n",
            "epoch 5 iter 639: train loss 1.45197. lr 5.990766e-04: 100%|██████████| 640/640 [07:34<00:00,  1.41it/s]\n",
            "epoch 6 iter 639: train loss 1.34232. lr 5.986703e-04: 100%|██████████| 640/640 [07:34<00:00,  1.41it/s]\n",
            "epoch 7 iter 639: train loss 1.22549. lr 5.981902e-04: 100%|██████████| 640/640 [07:34<00:00,  1.41it/s]\n",
            "epoch 8 iter 639: train loss 1.09058. lr 5.976366e-04: 100%|██████████| 640/640 [07:35<00:00,  1.40it/s]\n",
            "epoch 9 iter 639: train loss 0.92832. lr 5.970096e-04: 100%|██████████| 640/640 [07:35<00:00,  1.41it/s]\n",
            "epoch 10 iter 639: train loss 0.78630. lr 5.963092e-04: 100%|██████████| 640/640 [07:34<00:00,  1.41it/s]\n",
            "epoch 11 iter 639: train loss 0.63183. lr 5.955358e-04: 100%|██████████| 640/640 [07:35<00:00,  1.41it/s]\n",
            "epoch 12 iter 639: train loss 0.51112. lr 5.946894e-04: 100%|██████████| 640/640 [07:36<00:00,  1.40it/s]\n",
            "epoch 13 iter 639: train loss 0.38532. lr 5.937703e-04: 100%|██████████| 640/640 [07:36<00:00,  1.40it/s]\n",
            "epoch 14 iter 639: train loss 0.27902. lr 5.927788e-04: 100%|██████████| 640/640 [07:36<00:00,  1.40it/s]\n",
            "epoch 15 iter 639: train loss 0.22715. lr 5.917150e-04: 100%|██████████| 640/640 [07:36<00:00,  1.40it/s]\n",
            "epoch 16 iter 639: train loss 0.18122. lr 5.905792e-04: 100%|██████████| 640/640 [07:35<00:00,  1.40it/s]\n",
            "epoch 17 iter 639: train loss 0.15025. lr 5.893717e-04: 100%|██████████| 640/640 [07:34<00:00,  1.41it/s]\n",
            "epoch 18 iter 639: train loss 0.12516. lr 5.880928e-04: 100%|██████████| 640/640 [07:35<00:00,  1.41it/s]\n",
            "epoch 19 iter 639: train loss 0.11026. lr 5.867428e-04: 100%|██████████| 640/640 [07:35<00:00,  1.40it/s]\n",
            "epoch 20 iter 639: train loss 0.09605. lr 5.853221e-04: 100%|██████████| 640/640 [07:35<00:00,  1.40it/s]\n",
            "epoch 21 iter 639: train loss 0.09941. lr 5.838309e-04: 100%|██████████| 640/640 [07:35<00:00,  1.40it/s]\n",
            "epoch 22 iter 639: train loss 0.09065. lr 5.822698e-04: 100%|██████████| 640/640 [07:36<00:00,  1.40it/s]\n",
            "epoch 23 iter 639: train loss 0.07814. lr 5.806390e-04: 100%|██████████| 640/640 [07:37<00:00,  1.40it/s]\n",
            "epoch 24 iter 639: train loss 0.07309. lr 5.789389e-04: 100%|██████████| 640/640 [07:35<00:00,  1.41it/s]\n",
            "epoch 25 iter 639: train loss 0.08098. lr 5.771700e-04: 100%|██████████| 640/640 [07:33<00:00,  1.41it/s]\n",
            "epoch 26 iter 639: train loss 0.07272. lr 5.753327e-04: 100%|██████████| 640/640 [07:33<00:00,  1.41it/s]\n",
            "epoch 27 iter 639: train loss 0.07129. lr 5.734275e-04: 100%|██████████| 640/640 [07:34<00:00,  1.41it/s]\n",
            "epoch 28 iter 639: train loss 0.06380. lr 5.714549e-04: 100%|██████████| 640/640 [07:34<00:00,  1.41it/s]\n",
            "epoch 29 iter 639: train loss 0.06233. lr 5.694152e-04: 100%|██████████| 640/640 [07:33<00:00,  1.41it/s]\n",
            "epoch 30 iter 639: train loss 0.05897. lr 5.673091e-04: 100%|██████████| 640/640 [07:33<00:00,  1.41it/s]\n",
            "epoch 31 iter 639: train loss 0.05599. lr 5.651370e-04: 100%|██████████| 640/640 [07:33<00:00,  1.41it/s]\n",
            "epoch 32 iter 639: train loss 0.05085. lr 5.628995e-04: 100%|██████████| 640/640 [07:33<00:00,  1.41it/s]\n",
            "epoch 33 iter 639: train loss 0.05061. lr 5.605971e-04: 100%|██████████| 640/640 [07:33<00:00,  1.41it/s]\n",
            "epoch 34 iter 639: train loss 0.05761. lr 5.582304e-04: 100%|██████████| 640/640 [07:34<00:00,  1.41it/s]\n",
            "epoch 35 iter 639: train loss 0.04838. lr 5.558000e-04: 100%|██████████| 640/640 [07:35<00:00,  1.41it/s]\n",
            "epoch 36 iter 639: train loss 0.04536. lr 5.533065e-04: 100%|██████████| 640/640 [07:35<00:00,  1.41it/s]\n",
            "epoch 37 iter 639: train loss 0.04808. lr 5.507504e-04: 100%|██████████| 640/640 [07:35<00:00,  1.41it/s]\n",
            "epoch 38 iter 639: train loss 0.04685. lr 5.481326e-04: 100%|██████████| 640/640 [07:35<00:00,  1.41it/s]\n",
            "epoch 39 iter 639: train loss 0.04922. lr 5.454534e-04: 100%|██████████| 640/640 [07:35<00:00,  1.40it/s]\n",
            "epoch 40 iter 639: train loss 0.04511. lr 5.427138e-04: 100%|██████████| 640/640 [07:36<00:00,  1.40it/s]\n",
            "epoch 41 iter 639: train loss 0.04088. lr 5.399142e-04: 100%|██████████| 640/640 [07:36<00:00,  1.40it/s]\n",
            "epoch 42 iter 639: train loss 0.04118. lr 5.370554e-04: 100%|██████████| 640/640 [07:35<00:00,  1.40it/s]\n",
            "epoch 43 iter 639: train loss 0.04150. lr 5.341382e-04: 100%|██████████| 640/640 [07:36<00:00,  1.40it/s]\n",
            "epoch 44 iter 639: train loss 0.04204. lr 5.311631e-04: 100%|██████████| 640/640 [07:36<00:00,  1.40it/s]\n",
            "epoch 45 iter 5: train loss 0.09071. lr 5.311350e-04:   1%|          | 6/640 [00:04<07:31,  1.40it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-18-1d9d978c5f6f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m                       num_workers=4)\n\u001b[1;32m      7\u001b[0m \u001b[0mtrainer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrainer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtconf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/content/minGPT/mingpt/trainer.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    123\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_epochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 125\u001b[0;31m             \u001b[0mrun_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'train'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    126\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_dataset\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m                 \u001b[0mrun_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'test'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/minGPT/mingpt/trainer.py\u001b[0m in \u001b[0;36mrun_epoch\u001b[0;34m(split)\u001b[0m\n\u001b[1;32m     94\u001b[0m                     \u001b[0;31m# backprop and update the parameters\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m                     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m                     \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     97\u001b[0m                     \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip_grad_norm_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad_norm_clip\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m                     \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    183\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m         \"\"\"\n\u001b[0;32m--> 185\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    186\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m    125\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m    126\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 127\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    128\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t4FwyfR9Q8U3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "4644f01c-6313-41ee-8701-789f5c4e91cc"
      },
      "source": [
        "import joblib\n",
        "joblib.dump({'model' : model}, 'gpt_model.joblib', compress=0, protocol=None, cache_size=None)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['gpt_model.joblib']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qLHznMS18Drz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "15e6a5f2-fd8d-4ce3-ce29-b3055229b8cb"
      },
      "source": [
        "# alright, let's sample some character-level shakespear\n",
        "from mingpt.utils import sample\n",
        "\n",
        "context = \"\"\"sus pus\"\"\"\n",
        "x = torch.tensor([train_dataset.stoi[s] for s in context], dtype=torch.long)[None,...].to(trainer.device)\n",
        "y = sample(model, x, 100, temperature=0.9, sample=True, top_k=5)[0]\n",
        "completion = ''.join([train_dataset.itos[int(i)] for i in y])\n",
        "print(completion)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "sus pus olduğunun salamasına göre onlara olan olabilir fak ve sosyal başkalarıla 10-5 senedir lüsana kadar \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D2r_NK0-4Q13",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "8b25f5b7-d19f-44b2-8c32-9791eecc5ffc"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FXmu26Bq2T2X",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_dict = joblib.load('/content/drive/My Drive/models/gpt_model.joblib')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q03k4dam265r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_dict = a['model']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LWD8r44O3C5B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}