notebooks/microdistilmodel.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "microdistilmodel.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMtIaYav8rqRx4tHrcKmDn2",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gagan3012/project-code-py/blob/master/notebooks/microdistilmodel.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lsTTbIxoSXTc",
        "outputId": "41295c5a-52c3-4ae6-f0e8-83c3dd71d87a"
      },
      "source": [
        "lines = ['https://github.com/Garvit244/Leetcode',\r\n",
        "         'https://github.com/shichao-an/leetcode-python',\r\n",
        "         'https://github.com/algorhythms/LeetCode',\r\n",
        "         'https://github.com/wuduhren/leetcode-python',\r\n",
        "         'https://github.com/csujedihy/lc-all-solutions',\r\n",
        "         'https://github.com/vJechsmayr/PythonAlgorithms',\r\n",
        "         'https://github.com/HuberTRoy/leetCode',\r\n",
        "         'https://github.com/qiyuangong/leetcode',\r\n",
        "         'https://github.com/MTrajK/coding-problems',\r\n",
        "         'https://github.com/JushuangQiao/Python-LeetCode',\r\n",
        "         'https://github.com/Jack-Lee-Hiter/AlgorithmsByPython',\r\n",
        "         'https://github.com/sapanz/Hackerrank-Problem-Solving-Python-Solutions',\r\n",
        "         'https://github.com/arsho/Hackerrank_Python_Domain_Solutions',\r\n",
        "         'https://github.com/swapnanildutta/Hackerrank-Codes',\r\n",
        "         'https://github.com/markopuza/Competitive-programming-in-Python',\r\n",
        "         'https://github.com/deepaksood619/Python-Competitive-Programming',\r\n",
        "         'https://github.com/ndb796/Python-Competitive-Programming-Team-Notes',\r\n",
        "         'https://github.com/harshitbansal373/python',\r\n",
        "         'https://github.com/yashagrawal300/python-programs',\r\n",
        "         'https://github.com/bmegha98/Python-Practice',\r\n",
        "         'https://github.com/geekcomputers/Python',\r\n",
        "         'https://github.com/smilejay/python',\r\n",
        "         'https://github.com/yuzhoujr/leetcode',\r\n",
        "         'https://github.com/franklingu/leetcode-solutions',\r\n",
        "         'https://github.com/kumailn/Algorithms',\r\n",
        "         'https://github.com/Diego-Zulu/leetcode_answers',\r\n",
        "         'https://github.com/concealedtea/Coding-Interview-Prep',\r\n",
        "         'https://github.com/Wang-Yann/LeetCodeMe',\r\n",
        "         'https://github.com/hwm18/MyLeetCode',\r\n",
        "         'https://github.com/lixiang2017/leetcode',\r\n",
        "         'https://github.com/thisisshub/DSA',\r\n",
        "         'https://github.com/criszhou/LeetCode-Python',\r\n",
        "         'https://github.com/lilianweng/LeetcodePython',\r\n",
        "         'https://github.com/jioyoung/leetcode',\r\n",
        "         'https://github.com/Vikktour/Data-Structures-Algorithms-Implementations',\r\n",
        "         'https://github.com/lkwq007/leetcode-py',\r\n",
        "         'https://github.com/yz5308/Python_Leetcode',\r\n",
        "         'https://github.com/Garvit244/Leetcode',\r\n",
        "         'https://github.com/duanzhihao2017/Leetcode']\r\n",
        "\r\n",
        "len(lines)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "39"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_-BNj_gi_a8t"
      },
      "source": [
        "lines = ['https://github.com/Garvit244/Leetcode']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ArghELG6QZ5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9e4e8119-bbb7-4ecf-c307-6fe28f5da238"
      },
      "source": [
        "from subprocess import call\r\n",
        "import math\r\n",
        "import os\r\n",
        "import csv\r\n",
        "csv_columns = ['text']\r\n",
        "\r\n",
        "\r\n",
        "for line in lines:\r\n",
        "    call(['git', 'clone', line.strip(), f'resources/{line.strip().split(\"/\")[-1]}'])\r\n",
        "\r\n",
        "json_data = []\r\n",
        "total_files = []\r\n",
        "count = 0\r\n",
        "\r\n",
        "for line in lines:\r\n",
        "    for currentpath, folders, files in os.walk(f'resources/{line.strip().split(\"/\")[-1]}'):\r\n",
        "        for file in files:\r\n",
        "            if file[-3:] == '.py':\r\n",
        "                print(file)\r\n",
        "                count += 1\r\n",
        "                total_files.append(os.path.join(currentpath, file))\r\n",
        "\r\n",
        "print('files: ', len(total_files))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1091.py\n",
            "1277.py\n",
            "1296.py\n",
            "1268.py\n",
            "1275.py\n",
            "1267.py\n",
            "1290.py\n",
            "1200.py\n",
            "1282.py\n",
            "1291.py\n",
            "1276.py\n",
            "1266.py\n",
            "1283.py\n",
            "1295.py\n",
            "1281.py\n",
            "681.py\n",
            "674.py\n",
            "673.py\n",
            "1005.py\n",
            "1037.py\n",
            "1044.py\n",
            "1020.py\n",
            "1051.py\n",
            "1027.py\n",
            "1016.py\n",
            "1047.py\n",
            "1072.py\n",
            "1015.py\n",
            "1080.py\n",
            "1048.py\n",
            "1042.py\n",
            "1021.py\n",
            "1006.py\n",
            "1029.py\n",
            "1038.py\n",
            "1028.py\n",
            "1033.py\n",
            "1054.py\n",
            "1031.py\n",
            "1079.py\n",
            "1011.py\n",
            "1026.py\n",
            "1013.py\n",
            "1039.py\n",
            "1034.py\n",
            "1014.py\n",
            "1022.py\n",
            "1032.py\n",
            "1008.py\n",
            "1092.py\n",
            "1064.py\n",
            "1078.py\n",
            "1002.py\n",
            "1081.py\n",
            "1007.py\n",
            "1052.py\n",
            "1025.py\n",
            "1046.py\n",
            "1090.py\n",
            "1073.py\n",
            "1004.py\n",
            "1023.py\n",
            "1009.py\n",
            "1035.py\n",
            "1085.py\n",
            "1019.py\n",
            "1071.py\n",
            "1065.py\n",
            "1088.py\n",
            "1003.py\n",
            "1074.py\n",
            "1087.py\n",
            "1041.py\n",
            "1010.py\n",
            "1030.py\n",
            "1086.py\n",
            "1018.py\n",
            "1053.py\n",
            "1043.py\n",
            "1017.py\n",
            "1089.py\n",
            "387.py\n",
            "361.py\n",
            "301.py\n",
            "388.py\n",
            "350.py\n",
            "326.py\n",
            "315.py\n",
            "395.py\n",
            "322.py\n",
            "347.py\n",
            "334.py\n",
            "307.py\n",
            "329.py\n",
            "393.py\n",
            "346.py\n",
            "378.py\n",
            "332.py\n",
            "317.py\n",
            "328.py\n",
            "380.py\n",
            "351.py\n",
            "340.py\n",
            "995.py\n",
            "999.py\n",
            "991.py\n",
            "983.py\n",
            "926.py\n",
            "994.py\n",
            "990.py\n",
            "985.py\n",
            "998.py\n",
            "988.py\n",
            "993.py\n",
            "989.py\n",
            "981.py\n",
            "997.py\n",
            "977.py\n",
            "984.py\n",
            "831.py\n",
            "830.py\n",
            "1189.py\n",
            "1191.py\n",
            "1190.py\n",
            "1185.py\n",
            "1184.py\n",
            "1186.py\n",
            "297.py\n",
            "203.py\n",
            "210.py\n",
            "298.py\n",
            "234.py\n",
            "240.py\n",
            "283.py\n",
            "212.py\n",
            "295.py\n",
            "253.py\n",
            "279.py\n",
            "230.py\n",
            "236.py\n",
            "200.py\n",
            "281.py\n",
            "268.py\n",
            "257.py\n",
            "287.py\n",
            "218.py\n",
            "235.py\n",
            "208.py\n",
            "289.py\n",
            "300.py\n",
            "226.py\n",
            "238.py\n",
            "206.py\n",
            "285.py\n",
            "239.py\n",
            "215.py\n",
            "207.py\n",
            "448.py\n",
            "418.py\n",
            "442.py\n",
            "410.py\n",
            "482.py\n",
            "454.py\n",
            "116.py\n",
            "141.py\n",
            "105.py\n",
            "199.py\n",
            "112.py\n",
            "145.py\n",
            "134.py\n",
            "127.py\n",
            "140.py\n",
            "191.py\n",
            "120.py\n",
            "124.py\n",
            "173.py\n",
            "131.py\n",
            "155.py\n",
            "119.py\n",
            "115.py\n",
            "100.py\n",
            "101.py\n",
            "150.py\n",
            "102.py\n",
            "132.py\n",
            "190.py\n",
            "113.py\n",
            "146.py\n",
            "139.py\n",
            "129.py\n",
            "148.py\n",
            "170.py\n",
            "118.py\n",
            "152.py\n",
            "147.py\n",
            "143.py\n",
            "107.py\n",
            "159.py\n",
            "144.py\n",
            "160.py\n",
            "111.py\n",
            "163.py\n",
            "153.py\n",
            "179.py\n",
            "142.py\n",
            "106.py\n",
            "128.py\n",
            "103.py\n",
            "125.py\n",
            "130.py\n",
            "117.py\n",
            "108.py\n",
            "162.py\n",
            "123.py\n",
            "lcp.py\n",
            "suffix_array.py\n",
            "10.py\n",
            "75.py\n",
            "17.py\n",
            "97.py\n",
            "38.py\n",
            "95.py\n",
            "70.py\n",
            "05.py\n",
            "25.py\n",
            "40.py\n",
            "80.py\n",
            "TwoSum.py\n",
            "87.py\n",
            "85.py\n",
            "33.py\n",
            "36.py\n",
            "11.py\n",
            "19.py\n",
            "30.py\n",
            "92.py\n",
            "81.py\n",
            "90.py\n",
            "32.py\n",
            "60.py\n",
            "06.py\n",
            "86.py\n",
            "72.py\n",
            "56.py\n",
            "62.py\n",
            "14.py\n",
            "39.py\n",
            "45.py\n",
            "78.py\n",
            "57.py\n",
            "79.py\n",
            "82.py\n",
            "94.py\n",
            "61.py\n",
            "26.py\n",
            "03.py\n",
            "18.py\n",
            "73.py\n",
            "53.py\n",
            "93.py\n",
            "42.py\n",
            "66.py\n",
            "74.py\n",
            "15.py\n",
            "46.py\n",
            "99.py\n",
            "98.py\n",
            "63.py\n",
            "4.py\n",
            "16.py\n",
            "48.py\n",
            "23.py\n",
            "64.py\n",
            "83.py\n",
            "31.py\n",
            "8.py\n",
            "44.py\n",
            "67.py\n",
            "41.py\n",
            "54.py\n",
            "91.py\n",
            "65.py\n",
            "71.py\n",
            "22.py\n",
            "34.py\n",
            "24.py\n",
            "files:  286\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JR-q5mlSAWxJ"
      },
      "source": [
        "for file in total_files:\r\n",
        "    with open(file, \"r\") as f:\r\n",
        "        try:\r\n",
        "            t = f.readlines()\r\n",
        "        except UnicodeDecodeError:\r\n",
        "            print('DecoderError: ', file)\r\n",
        "        summary = ''.join(t)\r\n",
        "        summary = str(summary).strip()\r\n",
        "        bos_token = '<|title|>'\r\n",
        "        eos_token = '<|endoftext|>'\r\n",
        "        data = bos_token + summary + eos_token\r\n",
        "        json_data.append({'text': data})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JHdBhEpR6cfD"
      },
      "source": [
        "with open(\"data.csv\", 'w') as csvfile:\r\n",
        "        writer = csv.DictWriter(csvfile, fieldnames=csv_columns)\r\n",
        "        writer.writeheader()\r\n",
        "        for data in json_data:\r\n",
        "            writer.writerow(data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tH8MJYkm8Ldt"
      },
      "source": [
        "import pandas as pd\r\n",
        "\r\n",
        "df = pd.read_csv('/content/data.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oeTMr6_W-o4x",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 123
        },
        "outputId": "03d1714e-61af-4b9a-e9fd-be5a7e8c08ea"
      },
      "source": [
        "df['text'][1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'<|title|>\\'\\'\\'\\nGiven a m * n matrix of ones and zeros, return how many square submatrices have all ones.\\n\\n \\n\\nExample 1:\\n\\nInput: matrix =\\n[\\n  [0,1,1,1],\\n  [1,1,1,1],\\n  [0,1,1,1]\\n]\\nOutput: 15\\nExplanation: \\nThere are 10 squares of side 1.\\nThere are 4 squares of side 2.\\nThere is  1 square of side 3.\\nTotal number of squares = 10 + 4 + 1 = 15.\\nExample 2:\\n\\nInput: matrix = \\n[\\n  [1,0,1],\\n  [1,1,0],\\n  [1,1,0]\\n]\\nOutput: 7\\nExplanation: \\nThere are 6 squares of side 1.  \\nThere is 1 square of side 2. \\nTotal number of squares = 6 + 1 = 7.\\n \\n\\nConstraints:\\n\\n1 <= arr.length <= 300\\n1 <= arr[0].length <= 300\\n0 <= arr[i][j] <= 1\\n\\'\\'\\'\\nclass Solution(object):\\n    def countSquares(self, matrix):\\n        \"\"\"\\n        :type matrix: List[List[int]]\\n        :rtype: int\\n        \"\"\"  \\n    \\n        p_arr = [[0 for i in range(len(matrix[0]))] for j in range(len(matrix))]  \\n        result = 0\\n\\n        for index_i in range(1, len(matrix)):\\n            for index_j in range(1, len(matrix[0])):\\n                if matrix[index_i][index_j] == 1:\\n                    matrix[index_i][index_j] = min(matrix[index_i-1][index_j-1], min(matrix[index_i-1][index_j], matrix[index_i][index_j-1]))+1\\n        # print p_arr\\n        return sum([ sum(x) for x in matrix])<|endoftext|>'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EAmmNuZOUqeY",
        "outputId": "0be1e678-6317-4bde-dbd8-92cd0686ede9"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\r\n",
        "\r\n",
        "train, eval = train_test_split(df, train_size=.9, random_state=2020)\r\n",
        "print(len(train))\r\n",
        "print(len(eval))\r\n",
        "\r\n",
        "train = train['text'].tolist()\r\n",
        "eval = eval['text'].tolist()\r\n",
        "\r\n",
        "\r\n",
        "with open('train_tmp.txt', 'w') as file_handle:\r\n",
        "  file_handle.write(str(train))\r\n",
        "\r\n",
        "with open('eval_tmp.txt', 'w') as file_handle:\r\n",
        "  file_handle.write(str(eval))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "257\n",
            "29\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8JxD4TxQ-qCt",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "796980ba-029b-4bec-f61c-3b02cf1b6cad"
      },
      "source": [
        "!git clone https://github.com/huggingface/transformers\r\n",
        "!pip install transformers\r\n",
        "!pip install datasets\r\n",
        "!pip install wandb"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'transformers'...\n",
            "remote: Enumerating objects: 98, done.\u001b[K\n",
            "remote: Counting objects: 100% (98/98), done.\u001b[K\n",
            "remote: Compressing objects: 100% (72/72), done.\u001b[K\n",
            "remote: Total 66145 (delta 59), reused 49 (delta 24), pack-reused 66047\u001b[K\n",
            "Receiving objects: 100% (66145/66145), 49.55 MiB | 16.00 MiB/s, done.\n",
            "Resolving deltas: 100% (46952/46952), done.\n",
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f9/54/5ca07ec9569d2f232f3166de5457b63943882f7950ddfcc887732fc7fb23/transformers-4.3.3-py3-none-any.whl (1.9MB)\n",
            "\u001b[K     |████████████████████████████████| 1.9MB 8.1MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 36.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/23/2ddc317b2121117bf34dd00f5b0de194158f2a44ee2bf5e47c7166878a97/tokenizers-0.10.1-cp37-cp37m-manylinux2010_x86_64.whl (3.2MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 41.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp37-none-any.whl size=893262 sha256=507dbec43642f07c9a1baff5b52e26e33a8596e46846d3827a2cc266fabfa183\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.10.1 transformers-4.3.3\n",
            "Collecting datasets\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/3e/73/742d17d8a9a1c639132affccc9250f0743e484cbf263ede6ddcbe34ef212/datasets-1.4.1-py3-none-any.whl (186kB)\n",
            "\u001b[K     |████████████████████████████████| 194kB 7.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from datasets) (3.7.0)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.3)\n",
            "Requirement already satisfied: tqdm<4.50.0,>=4.27 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.41.1)\n",
            "Collecting fsspec\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/91/0d/a6bfee0ddf47b254286b9bd574e6f50978c69897647ae15b14230711806e/fsspec-0.8.7-py3-none-any.whl (103kB)\n",
            "\u001b[K     |████████████████████████████████| 112kB 13.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyarrow>=0.17.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n",
            "Collecting huggingface-hub==0.0.2\n",
            "  Downloading https://files.pythonhosted.org/packages/b5/93/7cb0755c62c36cdadc70c79a95681df685b52cbaf76c724facb6ecac3272/huggingface_hub-0.0.2-py3-none-any.whl\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.11.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.19.5)\n",
            "Collecting xxhash\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e7/27/1c0b37c53a7852f1c190ba5039404d27b3ae96a55f48203a74259f8213c9/xxhash-2.0.0-cp37-cp37m-manylinux2010_x86_64.whl (243kB)\n",
            "\u001b[K     |████████████████████████████████| 245kB 13.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->datasets) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->datasets) (3.4.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub==0.0.2->datasets) (3.0.12)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2020.12.5)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: fsspec, huggingface-hub, xxhash, datasets\n",
            "Successfully installed datasets-1.4.1 fsspec-0.8.7 huggingface-hub-0.0.2 xxhash-2.0.0\n",
            "Collecting wandb\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/00/17/b1e27f77c3d47f6915a774ecf632e3f5a7d49d9fa3991547729e7f19bedd/wandb-0.10.21-py2.py3-none-any.whl (2.0MB)\n",
            "\u001b[K     |████████████████████████████████| 2.0MB 8.3MB/s \n",
            "\u001b[?25hCollecting shortuuid>=0.5.0\n",
            "  Downloading https://files.pythonhosted.org/packages/25/a6/2ecc1daa6a304e7f1b216f0896b26156b78e7c38e1211e9b798b4716c53d/shortuuid-1.0.1-py3-none-any.whl\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from wandb) (3.13)\n",
            "Collecting subprocess32>=3.5.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/32/c8/564be4d12629b912ea431f1a50eb8b3b9d00f1a0b1ceff17f266be190007/subprocess32-3.5.4.tar.gz (97kB)\n",
            "\u001b[K     |████████████████████████████████| 102kB 14.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.23.0)\n",
            "Collecting configparser>=3.8.1\n",
            "  Downloading https://files.pythonhosted.org/packages/fd/01/ff260a18caaf4457eb028c96eeb405c4a230ca06c8ec9c1379f813caa52e/configparser-5.0.2-py3-none-any.whl\n",
            "Collecting pathtools\n",
            "  Downloading https://files.pythonhosted.org/packages/e7/7f/470d6fcdf23f9f3518f6b0b76be9df16dcc8630ad409947f8be2eb0ed13a/pathtools-0.1.2.tar.gz\n",
            "Collecting docker-pycreds>=0.4.0\n",
            "  Downloading https://files.pythonhosted.org/packages/f5/e8/f6bd1eee09314e7e6dee49cbe2c5e22314ccdb38db16c9fc72d2fa80d054/docker_pycreds-0.4.0-py2.py3-none-any.whl\n",
            "Collecting GitPython>=1.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/99/98019716955ba243657daedd1de8f3a88ca1f5b75057c38e959db22fb87b/GitPython-3.1.14-py3-none-any.whl (159kB)\n",
            "\u001b[K     |████████████████████████████████| 163kB 49.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (5.4.8)\n",
            "Collecting sentry-sdk>=0.4.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f3/92/5a33be64990ba815364a8f2dd9e6f51de60d23dfddafb4f1fc5577d4dc64/sentry_sdk-1.0.0-py2.py3-none-any.whl (131kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 37.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.8.1)\n",
            "Requirement already satisfied: Click>=7.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (7.1.2)\n",
            "Requirement already satisfied: six>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (1.15.0)\n",
            "Requirement already satisfied: protobuf>=3.12.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (3.12.4)\n",
            "Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (1.24.3)\n",
            "Collecting gitdb<5,>=4.0.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/48/11/d1800bca0a3bae820b84b7d813ad1eff15a48a64caea9c823fc8c1b119e8/gitdb-4.0.5-py3-none-any.whl (63kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 10.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.12.0->wandb) (54.0.0)\n",
            "Collecting smmap<4,>=3.0.1\n",
            "  Downloading https://files.pythonhosted.org/packages/d5/1e/6130925131f639b2acde0f7f18b73e33ce082ff2d90783c436b52040af5a/smmap-3.0.5-py2.py3-none-any.whl\n",
            "Building wheels for collected packages: subprocess32, pathtools\n",
            "  Building wheel for subprocess32 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for subprocess32: filename=subprocess32-3.5.4-cp37-none-any.whl size=6489 sha256=17ce13a106ade3c8734574497d4ddea6a5c307089ec3359eb792ab7c81381901\n",
            "  Stored in directory: /root/.cache/pip/wheels/68/39/1a/5e402bdfdf004af1786c8b853fd92f8c4a04f22aad179654d1\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-cp37-none-any.whl size=8786 sha256=e8428370aa5d99cad34fb79f4caf5ee7bda1988583aa123c4b91f1718a7b41e6\n",
            "  Stored in directory: /root/.cache/pip/wheels/0b/04/79/c3b0c3a0266a3cb4376da31e5bfe8bba0c489246968a68e843\n",
            "Successfully built subprocess32 pathtools\n",
            "Installing collected packages: shortuuid, subprocess32, configparser, pathtools, docker-pycreds, smmap, gitdb, GitPython, sentry-sdk, wandb\n",
            "Successfully installed GitPython-3.1.14 configparser-5.0.2 docker-pycreds-0.4.0 gitdb-4.0.5 pathtools-0.1.2 sentry-sdk-1.0.0 shortuuid-1.0.1 smmap-3.0.5 subprocess32-3.5.4 wandb-0.10.21\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ojIGLxZU1eK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fbfab763-d47b-455a-c75c-9367de5a307c"
      },
      "source": [
        "import os\r\n",
        "os.chdir(\"/content/transformers/\")\r\n",
        "!pip install .\r\n",
        "!pwd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Processing /content/transformers\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==4.4.0.dev0) (2.23.0)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers==4.4.0.dev0) (3.7.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.4.0.dev0) (2019.12.20)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.4.0.dev0) (1.19.5)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers==4.4.0.dev0) (20.9)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==4.4.0.dev0) (3.0.12)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==4.4.0.dev0) (4.41.1)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers==4.4.0.dev0) (0.10.1)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers==4.4.0.dev0) (0.0.43)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.4.0.dev0) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.4.0.dev0) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.4.0.dev0) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.4.0.dev0) (1.24.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers==4.4.0.dev0) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers==4.4.0.dev0) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers==4.4.0.dev0) (2.4.7)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.4.0.dev0) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.4.0.dev0) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.4.0.dev0) (1.0.1)\n",
            "Building wheels for collected packages: transformers\n",
            "  Building wheel for transformers (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-4.4.0.dev0-cp37-none-any.whl size=1938493 sha256=1fbbeda02fb06d14f713700dc2e1e8da0c4544c4b1010775f0cfae127236a843\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-opf0w196/wheels/23/19/dd/2561a4e47240cf6b307729d58e56f8077dd0c698f5992216cf\n",
            "Successfully built transformers\n",
            "Installing collected packages: transformers\n",
            "  Found existing installation: transformers 4.3.3\n",
            "    Uninstalling transformers-4.3.3:\n",
            "      Successfully uninstalled transformers-4.3.3\n",
            "Successfully installed transformers-4.4.0.dev0\n",
            "/content/transformers\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yhF9yjagBBNh",
        "outputId": "2f267bd1-780a-49f1-b172-1cb57cc74f5f"
      },
      "source": [
        "os.chdir(\"/content/transformers/examples/\")\r\n",
        "os.chdir(\"./language-modeling\")\r\n",
        "!pwd\r\n",
        "!pip install -r requirements.txt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/transformers/examples/language-modeling\n",
            "Requirement already satisfied: datasets>=1.1.3 in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 1)) (1.4.1)\n",
            "Collecting sentencepiece!=0.1.92\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f5/99/e0808cb947ba10f575839c43e8fafc9cc44e4a7a2c8f79c60db48220a577/sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 7.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 3)) (3.12.4)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (0.3.3)\n",
            "Requirement already satisfied: pyarrow>=0.17.1 in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (3.0.0)\n",
            "Requirement already satisfied: huggingface-hub==0.0.2 in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (0.0.2)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (2.0.0)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (2.23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (1.19.5)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (3.7.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (0.70.11.1)\n",
            "Requirement already satisfied: tqdm<4.50.0,>=4.27 in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (4.41.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (1.1.5)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (0.8.7)\n",
            "Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.7/dist-packages (from protobuf->-r requirements.txt (line 3)) (1.15.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf->-r requirements.txt (line 3)) (54.0.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub==0.0.2->datasets>=1.1.3->-r requirements.txt (line 1)) (3.0.12)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets>=1.1.3->-r requirements.txt (line 1)) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets>=1.1.3->-r requirements.txt (line 1)) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets>=1.1.3->-r requirements.txt (line 1)) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets>=1.1.3->-r requirements.txt (line 1)) (3.0.4)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->datasets>=1.1.3->-r requirements.txt (line 1)) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->datasets>=1.1.3->-r requirements.txt (line 1)) (3.7.4.3)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets>=1.1.3->-r requirements.txt (line 1)) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets>=1.1.3->-r requirements.txt (line 1)) (2018.9)\n",
            "Installing collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.95\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "id": "9Z51w4XLXjh-",
        "outputId": "f6cdd0e5-3494-4a9a-da88-c4121419ab45"
      },
      "source": [
        "import wandb\r\n",
        "wandb.login()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vh75u-DGFXaO",
        "outputId": "33a9a80b-a67a-4180-ebd2-44682f444800"
      },
      "source": [
        "%env WANDB_PROJECT=project-code-py\r\n",
        "\r\n",
        "!python run_clm.py \\\r\n",
        "--model_type distilgpt2 \\\r\n",
        "--model_name_or_path distilgpt2 \\\r\n",
        "--train_file \"/content/train_tmp.txt\" \\\r\n",
        "--do_train \\\r\n",
        "--validation_file \"/content/eval_tmp.txt\" \\\r\n",
        "--do_eval \\\r\n",
        "--per_device_train_batch_size 1 \\\r\n",
        "--per_device_eval_batch_size 1 \\\r\n",
        "--save_steps -1 \\\r\n",
        "--num_train_epochs 5 \\\r\n",
        "--fp16 \\\r\n",
        "--output_dir=\"/content/model\" \\\r\n",
        "--report_to wandb "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "env: WANDB_PROJECT=project-code-py\n",
            "2021-03-09 17:19:42.218795: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "03/09/2021 17:19:43 - WARNING - __main__ -   Process rank: -1, device: cuda:0, n_gpu: 1distributed training: False, 16-bits training: True\n",
            "03/09/2021 17:19:43 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir=/content/model, overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=IntervalStrategy.NO, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=5.0, max_steps=-1, lr_scheduler_type=SchedulerType.LINEAR, warmup_ratio=0.0, warmup_steps=0, logging_dir=runs/Mar09_17-19-43_1f5af2e94da2, logging_strategy=IntervalStrategy.STEPS, logging_first_step=False, logging_steps=500, save_strategy=IntervalStrategy.STEPS, save_steps=-1, save_total_limit=None, no_cuda=False, seed=42, fp16=True, fp16_opt_level=O1, fp16_backend=auto, fp16_full_eval=False, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=500, dataloader_num_workers=0, past_index=-1, run_name=/content/model, disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, report_to=['wandb'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, skip_memory_metrics=False, _n_gpu=1)\n",
            "03/09/2021 17:19:43 - WARNING - datasets.builder -   Using custom data configuration default-0d794a60fa7e5245\n",
            "Downloading and preparing dataset text/default (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/text/default-0d794a60fa7e5245/0.0.0/293ecb642f9fca45b44ad1f90c8445c54b9d80b95ab3fca3cfa5e1e3d85d4a57...\n",
            "Dataset text downloaded and prepared to /root/.cache/huggingface/datasets/text/default-0d794a60fa7e5245/0.0.0/293ecb642f9fca45b44ad1f90c8445c54b9d80b95ab3fca3cfa5e1e3d85d4a57. Subsequent calls will reuse this data.\n",
            "[INFO|file_utils.py:1371] 2021-03-09 17:19:43,956 >> https://huggingface.co/distilgpt2/resolve/main/config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpjr9sfs7m\n",
            "Downloading: 100% 762/762 [00:00<00:00, 790kB/s]\n",
            "[INFO|file_utils.py:1375] 2021-03-09 17:19:44,174 >> storing https://huggingface.co/distilgpt2/resolve/main/config.json in cache at /root/.cache/huggingface/transformers/f985248d2791fcff97732e4ee263617adec1edb5429a2b8421734c6d14e39bee.422318838d1ec4e061efb4ea29671cb2a044e244dc69229682bebd7cacc81631\n",
            "[INFO|file_utils.py:1378] 2021-03-09 17:19:44,174 >> creating metadata file for /root/.cache/huggingface/transformers/f985248d2791fcff97732e4ee263617adec1edb5429a2b8421734c6d14e39bee.422318838d1ec4e061efb4ea29671cb2a044e244dc69229682bebd7cacc81631\n",
            "[INFO|configuration_utils.py:463] 2021-03-09 17:19:44,175 >> loading configuration file https://huggingface.co/distilgpt2/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/f985248d2791fcff97732e4ee263617adec1edb5429a2b8421734c6d14e39bee.422318838d1ec4e061efb4ea29671cb2a044e244dc69229682bebd7cacc81631\n",
            "[INFO|configuration_utils.py:499] 2021-03-09 17:19:44,175 >> Model config GPT2Config {\n",
            "  \"_num_labels\": 1,\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0\n",
            "  },\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 6,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.4.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n",
            "[INFO|configuration_utils.py:463] 2021-03-09 17:19:44,379 >> loading configuration file https://huggingface.co/distilgpt2/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/f985248d2791fcff97732e4ee263617adec1edb5429a2b8421734c6d14e39bee.422318838d1ec4e061efb4ea29671cb2a044e244dc69229682bebd7cacc81631\n",
            "[INFO|configuration_utils.py:499] 2021-03-09 17:19:44,380 >> Model config GPT2Config {\n",
            "  \"_num_labels\": 1,\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0\n",
            "  },\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 6,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.4.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n",
            "[INFO|file_utils.py:1371] 2021-03-09 17:19:44,591 >> https://huggingface.co/distilgpt2/resolve/main/vocab.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpzgyx_k4i\n",
            "Downloading: 100% 1.04M/1.04M [00:00<00:00, 2.73MB/s]\n",
            "[INFO|file_utils.py:1375] 2021-03-09 17:19:45,184 >> storing https://huggingface.co/distilgpt2/resolve/main/vocab.json in cache at /root/.cache/huggingface/transformers/55051ac97dcc32f0a736d21a32a4d42b0d9b90f117ca7c38e65038b04bd5c3f5.c7ed1f96aac49e745788faa77ba0a26a392643a50bb388b9c04ff469e555241f\n",
            "[INFO|file_utils.py:1378] 2021-03-09 17:19:45,184 >> creating metadata file for /root/.cache/huggingface/transformers/55051ac97dcc32f0a736d21a32a4d42b0d9b90f117ca7c38e65038b04bd5c3f5.c7ed1f96aac49e745788faa77ba0a26a392643a50bb388b9c04ff469e555241f\n",
            "[INFO|file_utils.py:1371] 2021-03-09 17:19:45,394 >> https://huggingface.co/distilgpt2/resolve/main/merges.txt not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmphdh7b9tg\n",
            "Downloading: 100% 456k/456k [00:00<00:00, 1.45MB/s]\n",
            "[INFO|file_utils.py:1375] 2021-03-09 17:19:45,916 >> storing https://huggingface.co/distilgpt2/resolve/main/merges.txt in cache at /root/.cache/huggingface/transformers/9dfb299b74cdf7601ba7cd3a8073dbdac351caec0ed7ab5849b098b3c8ae3d57.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n",
            "[INFO|file_utils.py:1378] 2021-03-09 17:19:45,916 >> creating metadata file for /root/.cache/huggingface/transformers/9dfb299b74cdf7601ba7cd3a8073dbdac351caec0ed7ab5849b098b3c8ae3d57.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n",
            "[INFO|file_utils.py:1371] 2021-03-09 17:19:46,133 >> https://huggingface.co/distilgpt2/resolve/main/tokenizer.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpzb3dr7xa\n",
            "Downloading: 100% 1.36M/1.36M [00:00<00:00, 3.47MB/s]\n",
            "[INFO|file_utils.py:1375] 2021-03-09 17:19:46,740 >> storing https://huggingface.co/distilgpt2/resolve/main/tokenizer.json in cache at /root/.cache/huggingface/transformers/accb287b5a5396b2597382916b6cc939fdab1366e89475a92338d3971b3d02b7.cf2d0ecb83b6df91b3dbb53f1d1e4c311578bfd3aa0e04934215a49bf9898df0\n",
            "[INFO|file_utils.py:1378] 2021-03-09 17:19:46,740 >> creating metadata file for /root/.cache/huggingface/transformers/accb287b5a5396b2597382916b6cc939fdab1366e89475a92338d3971b3d02b7.cf2d0ecb83b6df91b3dbb53f1d1e4c311578bfd3aa0e04934215a49bf9898df0\n",
            "[INFO|tokenization_utils_base.py:1720] 2021-03-09 17:19:46,740 >> loading file https://huggingface.co/distilgpt2/resolve/main/vocab.json from cache at /root/.cache/huggingface/transformers/55051ac97dcc32f0a736d21a32a4d42b0d9b90f117ca7c38e65038b04bd5c3f5.c7ed1f96aac49e745788faa77ba0a26a392643a50bb388b9c04ff469e555241f\n",
            "[INFO|tokenization_utils_base.py:1720] 2021-03-09 17:19:46,740 >> loading file https://huggingface.co/distilgpt2/resolve/main/merges.txt from cache at /root/.cache/huggingface/transformers/9dfb299b74cdf7601ba7cd3a8073dbdac351caec0ed7ab5849b098b3c8ae3d57.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n",
            "[INFO|tokenization_utils_base.py:1720] 2021-03-09 17:19:46,740 >> loading file https://huggingface.co/distilgpt2/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/accb287b5a5396b2597382916b6cc939fdab1366e89475a92338d3971b3d02b7.cf2d0ecb83b6df91b3dbb53f1d1e4c311578bfd3aa0e04934215a49bf9898df0\n",
            "[INFO|file_utils.py:1371] 2021-03-09 17:19:47,010 >> https://huggingface.co/distilgpt2/resolve/main/pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp1sy3snqz\n",
            "Downloading: 100% 353M/353M [00:05<00:00, 68.6MB/s]\n",
            "[INFO|file_utils.py:1375] 2021-03-09 17:19:52,679 >> storing https://huggingface.co/distilgpt2/resolve/main/pytorch_model.bin in cache at /root/.cache/huggingface/transformers/43a212e83e76bcb07f45be584cf100676bdbbbe9c13f9e5c1c050049143a832f.a83d881ec4d624fd4b5826dd026e315246c48c67504ff91c0500570e291a54ba\n",
            "[INFO|file_utils.py:1378] 2021-03-09 17:19:52,679 >> creating metadata file for /root/.cache/huggingface/transformers/43a212e83e76bcb07f45be584cf100676bdbbbe9c13f9e5c1c050049143a832f.a83d881ec4d624fd4b5826dd026e315246c48c67504ff91c0500570e291a54ba\n",
            "[INFO|modeling_utils.py:1051] 2021-03-09 17:19:52,680 >> loading weights file https://huggingface.co/distilgpt2/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/43a212e83e76bcb07f45be584cf100676bdbbbe9c13f9e5c1c050049143a832f.a83d881ec4d624fd4b5826dd026e315246c48c67504ff91c0500570e291a54ba\n",
            "[INFO|modeling_utils.py:1167] 2021-03-09 17:19:55,795 >> All model checkpoint weights were used when initializing GPT2LMHeadModel.\n",
            "\n",
            "[INFO|modeling_utils.py:1176] 2021-03-09 17:19:55,796 >> All the weights of GPT2LMHeadModel were initialized from the model checkpoint at distilgpt2.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use GPT2LMHeadModel for predictions without further training.\n",
            "[WARNING|tokenization_utils_base.py:3152] 2021-03-09 17:19:56,064 >> Token indices sequence length is longer than the specified maximum sequence length for this model (164604 > 1024). Running this sequence through the model will result in indexing errors\n",
            "100% 1/1 [00:00<00:00,  3.89ba/s]\n",
            "100% 1/1 [00:00<00:00, 34.50ba/s]\n",
            "100% 1/1 [00:00<00:00,  7.70ba/s]\n",
            "100% 1/1 [00:00<00:00, 68.17ba/s]\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "[INFO|trainer.py:384] 2021-03-09 17:20:07,740 >> Using amp fp16 backend\n",
            "[INFO|trainer.py:935] 2021-03-09 17:20:07,954 >> ***** Running training *****\n",
            "[INFO|trainer.py:936] 2021-03-09 17:20:07,955 >>   Num examples = 160\n",
            "[INFO|trainer.py:937] 2021-03-09 17:20:07,955 >>   Num Epochs = 5\n",
            "[INFO|trainer.py:938] 2021-03-09 17:20:07,955 >>   Instantaneous batch size per device = 1\n",
            "[INFO|trainer.py:939] 2021-03-09 17:20:07,955 >>   Total train batch size (w. parallel, distributed & accumulation) = 1\n",
            "[INFO|trainer.py:940] 2021-03-09 17:20:07,955 >>   Gradient Accumulation steps = 1\n",
            "[INFO|trainer.py:941] 2021-03-09 17:20:07,955 >>   Total optimization steps = 800\n",
            "[INFO|integrations.py:557] 2021-03-09 17:20:07,956 >> Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mgagan3012\u001b[0m (use `wandb login --relogin` to force relogin)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "2021-03-09 17:20:09.079531: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.10.21\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33m/content/model\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/gagan3012/project-code-py\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/gagan3012/project-code-py/runs/b6fg1q8v\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in /content/transformers/examples/language-modeling/wandb/run-20210309_172008-b6fg1q8v\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run `wandb offline` to turn off syncing.\n",
            "\n",
            "  0%|          | 0/800 [00:00<?, ?it/s]/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py:760: UserWarning: Using non-full backward hooks on a Module that does not return a single Tensor or a tuple of Tensors is deprecated and will be removed in future versions. This hook will be missing some of the grad_output. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using non-full backward hooks on a Module that does not return a \"\n",
            "/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py:795: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
            "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n",
            "/usr/local/lib/python3.7/dist-packages/torch/optim/lr_scheduler.py:134: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
            "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n",
            " 62%|██████▎   | 500/800 [01:29<01:04,  4.67it/s]{'loss': 1.419, 'learning_rate': 1.8750000000000002e-05, 'epoch': 3.12}\n",
            "100%|██████████| 800/800 [02:22<00:00,  5.54it/s][INFO|trainer.py:1118] 2021-03-09 17:22:33,120 >> \n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "                                                 \n",
            "100%|██████████| 800/800 [02:22<00:00,  5.60it/s]\n",
            "[INFO|trainer.py:1538] 2021-03-09 17:22:33,300 >> Saving model checkpoint to /content/model\n",
            "[INFO|configuration_utils.py:314] 2021-03-09 17:22:33,301 >> Configuration saved in /content/model/config.json\n",
            "[INFO|modeling_utils.py:837] 2021-03-09 17:22:34,048 >> Model weights saved in /content/model/pytorch_model.bin\n",
            "[INFO|tokenization_utils_base.py:1914] 2021-03-09 17:22:34,050 >> tokenizer config file saved in /content/model/tokenizer_config.json\n",
            "[INFO|tokenization_utils_base.py:1920] 2021-03-09 17:22:34,050 >> Special tokens file saved in /content/model/special_tokens_map.json\n",
            "[INFO|trainer_pt_utils.py:622] 2021-03-09 17:22:34,122 >> ***** train metrics *****\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:34,122 >>   epoch                      =      5.0\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:34,122 >>   init_mem_cpu_alloc_delta   =      9MB\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:34,122 >>   init_mem_cpu_peaked_delta  =      0MB\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:34,122 >>   init_mem_gpu_alloc_delta   =    319MB\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:34,122 >>   init_mem_gpu_peaked_delta  =      0MB\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:34,122 >>   train_mem_cpu_alloc_delta  =      1MB\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:34,122 >>   train_mem_cpu_peaked_delta =      0MB\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:34,122 >>   train_mem_gpu_alloc_delta  =    941MB\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:34,122 >>   train_mem_gpu_peaked_delta =   1942MB\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:34,122 >>   train_runtime              = 145.1649\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:34,122 >>   train_samples              =      160\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:34,122 >>   train_samples_per_second   =    5.511\n",
            "03/09/2021 17:22:34 - INFO - __main__ -   *** Evaluate ***\n",
            "[INFO|trainer.py:1752] 2021-03-09 17:22:34,237 >> ***** Running Evaluation *****\n",
            "[INFO|trainer.py:1753] 2021-03-09 17:22:34,237 >>   Num examples = 18\n",
            "[INFO|trainer.py:1754] 2021-03-09 17:22:34,238 >>   Batch size = 1\n",
            " 94%|█████████▍| 17/18 [00:01<00:00, 12.19it/s]wandb: WARNING Step must only increase in log calls.  Step 800 < 801; dropping {'eval/loss': 1.1778761148452759, 'eval/runtime': 1.5089, 'eval/samples_per_second': 11.929, 'train/epoch': 5.0}.\n",
            "100%|██████████| 18/18 [00:01<00:00, 12.09it/s]\n",
            "[INFO|trainer_pt_utils.py:622] 2021-03-09 17:22:35,859 >> ***** eval metrics *****\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:35,860 >>   epoch                     =    5.0\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:35,860 >>   eval_loss                 = 1.1779\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:35,861 >>   eval_mem_cpu_alloc_delta  =    0MB\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:35,861 >>   eval_mem_cpu_peaked_delta =    0MB\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:35,861 >>   eval_mem_gpu_alloc_delta  =    0MB\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:35,861 >>   eval_mem_gpu_peaked_delta =  449MB\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:35,862 >>   eval_runtime              = 1.5089\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:35,862 >>   eval_samples              =     18\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:35,862 >>   eval_samples_per_second   = 11.929\n",
            "[INFO|trainer_pt_utils.py:627] 2021-03-09 17:22:35,862 >>   perplexity                = 3.2475\n",
            "\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Waiting for W&B process to finish, PID 243\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Program ended successfully.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find user logs for this run at: /content/transformers/examples/language-modeling/wandb/run-20210309_172008-b6fg1q8v/logs/debug.log\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find internal logs for this run at: /content/transformers/examples/language-modeling/wandb/run-20210309_172008-b6fg1q8v/logs/debug-internal.log\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                       _runtime 145\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                     _timestamp 1615310553\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                          _step 800\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                     train/loss 1.419\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                            train/learning_rate 2e-05\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                    train/epoch 5.0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                            train/train_runtime 145.1649\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                 train/train_samples_per_second 5.511\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                               train/total_flos 402616693555200.0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                         _runtime ▁██\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                       _timestamp ▁██\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                            _step ▁▅█\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                       train/loss ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:              train/learning_rate ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                      train/epoch ▁█\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:              train/train_runtime ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   train/train_samples_per_second ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                 train/total_flos ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 1 media file(s), 0 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced \u001b[33m/content/model\u001b[0m: \u001b[34mhttps://wandb.ai/gagan3012/project-code-py/runs/b6fg1q8v\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}