kaggle_bert.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "kaggle_bert.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "tNC1nWvooP8i",
        "colab_type": "code",
        "outputId": "d5e7cfa9-06f8-4dbe-904c-55a2bdbc3167",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        }
      },
      "source": [
        "! git config --global user.email \"jordimoreratrujillo@gmail.com\"\n",
        "! git config --global user.name \"jorditruji\"\n",
        "! git clone https://github.com/jorditruji/BERT_kaggle.git\n",
        "!pip install pytorch_pretrained_bert\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fatal: destination path 'BERT_kaggle' already exists and is not an empty directory.\n",
            "Requirement already satisfied: pytorch_pretrained_bert in /usr/local/lib/python3.6/dist-packages (0.6.2)\n",
            "Requirement already satisfied: torch>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (1.1.0)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (2018.1.10)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (1.16.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (2.21.0)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (1.9.140)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (4.28.1)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch_pretrained_bert) (2.8)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch_pretrained_bert) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch_pretrained_bert) (2019.3.9)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch_pretrained_bert) (1.24.2)\n",
            "Requirement already satisfied: botocore<1.13.0,>=1.12.140 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch_pretrained_bert) (1.12.140)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch_pretrained_bert) (0.9.4)\n",
            "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch_pretrained_bert) (0.2.0)\n",
            "Requirement already satisfied: docutils>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.13.0,>=1.12.140->boto3->pytorch_pretrained_bert) (0.14)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\" in /usr/local/lib/python3.6/dist-packages (from botocore<1.13.0,>=1.12.140->boto3->pytorch_pretrained_bert) (2.5.3)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\"->botocore<1.13.0,>=1.12.140->boto3->pytorch_pretrained_bert) (1.12.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WZjyf5jIp3sX",
        "colab_type": "code",
        "outputId": "a9fbb677-3ef0-4812-e1b3-a4f79d475cd8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "!ls"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "BERT_kaggle  sample_data  train.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xQ3-9eRIp3wv",
        "colab_type": "code",
        "outputId": "17ed55ea-732d-4bc9-9f2c-54f86d0c9022",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 4048
        }
      },
      "source": [
        "\n",
        "from __future__ import absolute_import, division, print_function\n",
        "\n",
        "import argparse\n",
        "import csv\n",
        "import logging\n",
        "import os\n",
        "import random\n",
        "import sys\n",
        "import pandas as pd \n",
        "import numpy as np\n",
        "import re\n",
        "import torch\n",
        "\n",
        "from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler,\n",
        "                              TensorDataset)\n",
        "from torch.utils.data.distributed import DistributedSampler\n",
        "from tqdm import tqdm, trange\n",
        "\n",
        "from torch.nn import CrossEntropyLoss, MSELoss\n",
        "from scipy.stats import pearsonr, spearmanr\n",
        "from sklearn.metrics import matthews_corrcoef, f1_score\n",
        "\n",
        "from pytorch_pretrained_bert.file_utils import PYTORCH_PRETRAINED_BERT_CACHE, WEIGHTS_NAME, CONFIG_NAME\n",
        "from pytorch_pretrained_bert.modeling import BertForSequenceClassification, BertConfig\n",
        "from pytorch_pretrained_bert.tokenization import BertTokenizer\n",
        "from pytorch_pretrained_bert.optimization import BertAdam, WarmupLinearSchedule\n",
        "\n",
        "from BERT_kaggle.Data_management.data_helpers import InputFeatures, InputExample, convert_examples_to_features, read_examples\n",
        "\n",
        "\n",
        "logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s', \n",
        "                    datefmt = '%m/%d/%Y %H:%M:%S',\n",
        "                    level = logging.INFO)\n",
        "logger = logging.getLogger(__name__)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "train, labels, toxicity = read_examples('train.csv')\n",
        "\n",
        "#train = pd.read_csv('../../Datasets/kaggle/train.csv', index_col='id')\n",
        "#test = pd.read_csv('../input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', index_col='id')\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "#device = 'cpu'\n",
        "n_gpu = 1\n",
        "# Bert tokenizer\n",
        "maxlen = 80\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case= True)\n",
        "num_labels = 2\n",
        "# Regress or classify\n",
        "mode = 'classification'\n",
        "#mode = 'regression'\n",
        "train_features = convert_examples_to_features(train, [\"OK\", \"Toxic\"], maxlen, tokenizer,mode )\n",
        "\n",
        "\n",
        "all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)\n",
        "all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)\n",
        "all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)\n"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "05/09/2019 17:37:41 - INFO - pytorch_pretrained_bert.tokenization -   loading vocabulary file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /root/.pytorch_pretrained_bert/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 0 of 1804874\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   *** Example ***\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   guid: 0\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   tokens: [CLS] this is so cool . it ' s like , ' would you want your mother to read this ? ? ' really great idea , well done ! [SEP]\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   input_ids: 101 2023 2003 2061 4658 1012 2009 1005 1055 2066 1010 1005 2052 2017 2215 2115 2388 2000 3191 2023 1029 1029 1005 2428 2307 2801 1010 2092 2589 999 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   label: OK (id = 0)\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   *** Example ***\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   guid: 1\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   tokens: [CLS] thank you ! ! this would make my life a lot less anxiety - inducing . keep it up , and don ' t let anyone get in your way ! [SEP]\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   input_ids: 101 4067 2017 999 999 2023 2052 2191 2026 2166 1037 2843 2625 10089 1011 29290 1012 2562 2009 2039 1010 1998 2123 1005 1056 2292 3087 2131 1999 2115 2126 999 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   label: OK (id = 0)\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   *** Example ***\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   guid: 2\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   tokens: [CLS] this is such an urgent design problem ; ku ##dos to you for taking it on . very impressive ! [SEP]\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   input_ids: 101 2023 2003 2107 2019 13661 2640 3291 1025 13970 12269 2000 2017 2005 2635 2009 2006 1012 2200 8052 999 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   label: OK (id = 0)\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   *** Example ***\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   guid: 3\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   tokens: [CLS] is this something i ' ll be able to install on my site ? when will you be releasing it ? [SEP]\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   input_ids: 101 2003 2023 2242 1045 1005 2222 2022 2583 2000 16500 2006 2026 2609 1029 2043 2097 2017 2022 8287 2009 1029 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   label: OK (id = 0)\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   *** Example ***\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   guid: 4\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   tokens: [CLS] ha ##ha you guys are a bunch of losers . [SEP]\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   input_ids: 101 5292 3270 2017 4364 2024 1037 9129 1997 23160 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            "05/09/2019 17:37:41 - INFO - BERT_kaggle.Data_management.data_helpers -   label: Toxic (id = 1)\n",
            "05/09/2019 17:37:56 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 10000 of 1804874\n",
            "05/09/2019 17:38:08 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 20000 of 1804874\n",
            "05/09/2019 17:38:20 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 30000 of 1804874\n",
            "05/09/2019 17:38:32 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 40000 of 1804874\n",
            "05/09/2019 17:38:44 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 50000 of 1804874\n",
            "05/09/2019 17:38:56 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 60000 of 1804874\n",
            "05/09/2019 17:39:08 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 70000 of 1804874\n",
            "05/09/2019 17:39:20 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 80000 of 1804874\n",
            "05/09/2019 17:39:32 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 90000 of 1804874\n",
            "05/09/2019 17:39:45 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 100000 of 1804874\n",
            "05/09/2019 17:39:58 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 110000 of 1804874\n",
            "05/09/2019 17:40:11 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 120000 of 1804874\n",
            "05/09/2019 17:40:23 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 130000 of 1804874\n",
            "05/09/2019 17:40:34 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 140000 of 1804874\n",
            "05/09/2019 17:40:47 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 150000 of 1804874\n",
            "05/09/2019 17:40:59 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 160000 of 1804874\n",
            "05/09/2019 17:41:12 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 170000 of 1804874\n",
            "05/09/2019 17:41:24 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 180000 of 1804874\n",
            "05/09/2019 17:41:36 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 190000 of 1804874\n",
            "05/09/2019 17:41:49 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 200000 of 1804874\n",
            "05/09/2019 17:42:02 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 210000 of 1804874\n",
            "05/09/2019 17:42:14 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 220000 of 1804874\n",
            "05/09/2019 17:42:26 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 230000 of 1804874\n",
            "05/09/2019 17:42:40 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 240000 of 1804874\n",
            "05/09/2019 17:42:53 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 250000 of 1804874\n",
            "05/09/2019 17:43:06 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 260000 of 1804874\n",
            "05/09/2019 17:43:18 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 270000 of 1804874\n",
            "05/09/2019 17:43:31 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 280000 of 1804874\n",
            "05/09/2019 17:43:42 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 290000 of 1804874\n",
            "05/09/2019 17:43:55 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 300000 of 1804874\n",
            "05/09/2019 17:44:07 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 310000 of 1804874\n",
            "05/09/2019 17:44:20 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 320000 of 1804874\n",
            "05/09/2019 17:44:32 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 330000 of 1804874\n",
            "05/09/2019 17:44:44 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 340000 of 1804874\n",
            "05/09/2019 17:44:56 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 350000 of 1804874\n",
            "05/09/2019 17:45:08 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 360000 of 1804874\n",
            "05/09/2019 17:45:19 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 370000 of 1804874\n",
            "05/09/2019 17:45:31 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 380000 of 1804874\n",
            "05/09/2019 17:45:43 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 390000 of 1804874\n",
            "05/09/2019 17:45:57 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 400000 of 1804874\n",
            "05/09/2019 17:46:08 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 410000 of 1804874\n",
            "05/09/2019 17:46:20 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 420000 of 1804874\n",
            "05/09/2019 17:46:32 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 430000 of 1804874\n",
            "05/09/2019 17:46:44 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 440000 of 1804874\n",
            "05/09/2019 17:46:55 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 450000 of 1804874\n",
            "05/09/2019 17:47:07 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 460000 of 1804874\n",
            "05/09/2019 17:47:19 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 470000 of 1804874\n",
            "05/09/2019 17:47:30 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 480000 of 1804874\n",
            "05/09/2019 17:47:42 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 490000 of 1804874\n",
            "05/09/2019 17:47:54 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 500000 of 1804874\n",
            "05/09/2019 17:48:05 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 510000 of 1804874\n",
            "05/09/2019 17:48:16 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 520000 of 1804874\n",
            "05/09/2019 17:48:27 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 530000 of 1804874\n",
            "05/09/2019 17:48:39 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 540000 of 1804874\n",
            "05/09/2019 17:48:50 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 550000 of 1804874\n",
            "05/09/2019 17:49:02 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 560000 of 1804874\n",
            "05/09/2019 17:49:14 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 570000 of 1804874\n",
            "05/09/2019 17:49:26 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 580000 of 1804874\n",
            "05/09/2019 17:49:38 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 590000 of 1804874\n",
            "05/09/2019 17:49:52 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 600000 of 1804874\n",
            "05/09/2019 17:50:04 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 610000 of 1804874\n",
            "05/09/2019 17:50:16 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 620000 of 1804874\n",
            "05/09/2019 17:50:28 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 630000 of 1804874\n",
            "05/09/2019 17:50:41 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 640000 of 1804874\n",
            "05/09/2019 17:50:53 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 650000 of 1804874\n",
            "05/09/2019 17:51:05 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 660000 of 1804874\n",
            "05/09/2019 17:51:17 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 670000 of 1804874\n",
            "05/09/2019 17:51:29 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 680000 of 1804874\n",
            "05/09/2019 17:51:41 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 690000 of 1804874\n",
            "05/09/2019 17:51:53 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 700000 of 1804874\n",
            "05/09/2019 17:52:05 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 710000 of 1804874\n",
            "05/09/2019 17:52:17 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 720000 of 1804874\n",
            "05/09/2019 17:52:29 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 730000 of 1804874\n",
            "05/09/2019 17:52:41 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 740000 of 1804874\n",
            "05/09/2019 17:52:53 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 750000 of 1804874\n",
            "05/09/2019 17:53:05 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 760000 of 1804874\n",
            "05/09/2019 17:53:17 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 770000 of 1804874\n",
            "05/09/2019 17:53:29 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 780000 of 1804874\n",
            "05/09/2019 17:53:42 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 790000 of 1804874\n",
            "05/09/2019 17:53:53 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 800000 of 1804874\n",
            "05/09/2019 17:54:05 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 810000 of 1804874\n",
            "05/09/2019 17:54:17 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 820000 of 1804874\n",
            "05/09/2019 17:54:29 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 830000 of 1804874\n",
            "05/09/2019 17:54:45 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 840000 of 1804874\n",
            "05/09/2019 17:54:56 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 850000 of 1804874\n",
            "05/09/2019 17:55:09 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 860000 of 1804874\n",
            "05/09/2019 17:55:21 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 870000 of 1804874\n",
            "05/09/2019 17:55:33 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 880000 of 1804874\n",
            "05/09/2019 17:55:45 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 890000 of 1804874\n",
            "05/09/2019 17:55:58 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 900000 of 1804874\n",
            "05/09/2019 17:56:10 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 910000 of 1804874\n",
            "05/09/2019 17:56:22 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 920000 of 1804874\n",
            "05/09/2019 17:56:35 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 930000 of 1804874\n",
            "05/09/2019 17:56:47 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 940000 of 1804874\n",
            "05/09/2019 17:56:58 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 950000 of 1804874\n",
            "05/09/2019 17:57:10 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 960000 of 1804874\n",
            "05/09/2019 17:57:22 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 970000 of 1804874\n",
            "05/09/2019 17:57:33 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 980000 of 1804874\n",
            "05/09/2019 17:57:45 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 990000 of 1804874\n",
            "05/09/2019 17:57:57 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1000000 of 1804874\n",
            "05/09/2019 17:58:08 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1010000 of 1804874\n",
            "05/09/2019 17:58:19 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1020000 of 1804874\n",
            "05/09/2019 17:58:31 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1030000 of 1804874\n",
            "05/09/2019 17:58:42 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1040000 of 1804874\n",
            "05/09/2019 17:58:54 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1050000 of 1804874\n",
            "05/09/2019 17:59:06 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1060000 of 1804874\n",
            "05/09/2019 17:59:17 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1070000 of 1804874\n",
            "05/09/2019 17:59:29 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1080000 of 1804874\n",
            "05/09/2019 17:59:40 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1090000 of 1804874\n",
            "05/09/2019 17:59:52 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1100000 of 1804874\n",
            "05/09/2019 18:00:04 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1110000 of 1804874\n",
            "05/09/2019 18:00:16 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1120000 of 1804874\n",
            "05/09/2019 18:00:27 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1130000 of 1804874\n",
            "05/09/2019 18:00:38 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1140000 of 1804874\n",
            "05/09/2019 18:00:55 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1150000 of 1804874\n",
            "05/09/2019 18:01:06 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1160000 of 1804874\n",
            "05/09/2019 18:01:18 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1170000 of 1804874\n",
            "05/09/2019 18:01:29 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1180000 of 1804874\n",
            "05/09/2019 18:01:40 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1190000 of 1804874\n",
            "05/09/2019 18:01:51 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1200000 of 1804874\n",
            "05/09/2019 18:02:03 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1210000 of 1804874\n",
            "05/09/2019 18:02:14 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1220000 of 1804874\n",
            "05/09/2019 18:02:26 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1230000 of 1804874\n",
            "05/09/2019 18:02:37 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1240000 of 1804874\n",
            "05/09/2019 18:02:49 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1250000 of 1804874\n",
            "05/09/2019 18:03:01 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1260000 of 1804874\n",
            "05/09/2019 18:03:12 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1270000 of 1804874\n",
            "05/09/2019 18:03:23 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1280000 of 1804874\n",
            "05/09/2019 18:03:35 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1290000 of 1804874\n",
            "05/09/2019 18:03:46 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1300000 of 1804874\n",
            "05/09/2019 18:03:57 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1310000 of 1804874\n",
            "05/09/2019 18:04:09 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1320000 of 1804874\n",
            "05/09/2019 18:04:21 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1330000 of 1804874\n",
            "05/09/2019 18:04:33 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1340000 of 1804874\n",
            "05/09/2019 18:04:44 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1350000 of 1804874\n",
            "05/09/2019 18:04:55 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1360000 of 1804874\n",
            "05/09/2019 18:05:07 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1370000 of 1804874\n",
            "05/09/2019 18:05:19 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1380000 of 1804874\n",
            "05/09/2019 18:05:31 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1390000 of 1804874\n",
            "05/09/2019 18:05:42 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1400000 of 1804874\n",
            "05/09/2019 18:05:54 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1410000 of 1804874\n",
            "05/09/2019 18:06:05 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1420000 of 1804874\n",
            "05/09/2019 18:06:16 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1430000 of 1804874\n",
            "05/09/2019 18:06:27 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1440000 of 1804874\n",
            "05/09/2019 18:06:39 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1450000 of 1804874\n",
            "05/09/2019 18:06:51 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1460000 of 1804874\n",
            "05/09/2019 18:07:03 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1470000 of 1804874\n",
            "05/09/2019 18:07:14 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1480000 of 1804874\n",
            "05/09/2019 18:07:26 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1490000 of 1804874\n",
            "05/09/2019 18:07:37 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1500000 of 1804874\n",
            "05/09/2019 18:07:49 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1510000 of 1804874\n",
            "05/09/2019 18:08:01 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1520000 of 1804874\n",
            "05/09/2019 18:08:12 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1530000 of 1804874\n",
            "05/09/2019 18:08:30 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1540000 of 1804874\n",
            "05/09/2019 18:08:41 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1550000 of 1804874\n",
            "05/09/2019 18:08:53 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1560000 of 1804874\n",
            "05/09/2019 18:09:04 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1570000 of 1804874\n",
            "05/09/2019 18:09:15 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1580000 of 1804874\n",
            "05/09/2019 18:09:27 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1590000 of 1804874\n",
            "05/09/2019 18:09:38 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1600000 of 1804874\n",
            "05/09/2019 18:09:50 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1610000 of 1804874\n",
            "05/09/2019 18:10:02 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1620000 of 1804874\n",
            "05/09/2019 18:10:14 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1630000 of 1804874\n",
            "05/09/2019 18:10:25 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1640000 of 1804874\n",
            "05/09/2019 18:10:37 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1650000 of 1804874\n",
            "05/09/2019 18:10:48 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1660000 of 1804874\n",
            "05/09/2019 18:10:59 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1670000 of 1804874\n",
            "05/09/2019 18:11:10 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1680000 of 1804874\n",
            "05/09/2019 18:11:22 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1690000 of 1804874\n",
            "05/09/2019 18:11:33 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1700000 of 1804874\n",
            "05/09/2019 18:11:45 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1710000 of 1804874\n",
            "05/09/2019 18:11:56 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1720000 of 1804874\n",
            "05/09/2019 18:12:08 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1730000 of 1804874\n",
            "05/09/2019 18:12:19 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1740000 of 1804874\n",
            "05/09/2019 18:12:31 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1750000 of 1804874\n",
            "05/09/2019 18:12:42 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1760000 of 1804874\n",
            "05/09/2019 18:12:53 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1770000 of 1804874\n",
            "05/09/2019 18:13:04 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1780000 of 1804874\n",
            "05/09/2019 18:13:16 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1790000 of 1804874\n",
            "05/09/2019 18:13:27 - INFO - BERT_kaggle.Data_management.data_helpers -   Writing example 1800000 of 1804874\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wDAksRI0p30p",
        "colab_type": "code",
        "outputId": "abb41222-6711-4020-9c5a-eb86878d2a53",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 220
        }
      },
      "source": [
        "\n",
        "'''\n",
        "if mode == \"classification\":\n",
        "    all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)\n",
        "elif mode == \"regression\":\n",
        "    all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.float)\n",
        "\n",
        "train_features = []\n",
        "del(train_features)\n",
        "    \n",
        "train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "batch_size = 16\n",
        "\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "\n",
        "model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)\n",
        "param_optimizer = list(model.named_parameters())\n",
        "no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n",
        "optimizer_grouped_parameters = [\n",
        "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n",
        "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
        "\t]\n",
        "'''\n",
        "\n",
        "num_train_epochs = 4\n",
        "gradient_accumulation_steps = 1\n",
        "num_train_optimization_steps = int(len(train) / batch_size ) * num_train_epochs\n",
        "print(num_train_optimization_steps)\n",
        "\n",
        "optimizer = BertAdam(optimizer_grouped_parameters,\n",
        "                     lr=5e-5,\n",
        "                     warmup=0.1,\n",
        "                     t_total=num_train_optimization_steps)\n",
        "global_step = 0\n",
        "nb_tr_steps = 0\n",
        "tr_loss = 0\n",
        "model.train()\n",
        "model.to(device)\n",
        "for _ in trange(int(num_train_epochs), desc=\"Epoch\"):\n",
        "    running_corrects = 0\n",
        "    tr_loss = 0\n",
        "    nb_tr_examples, nb_tr_steps = 0, 0\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "        batch = tuple(t.to(device) for t in batch)\n",
        "        input_ids, input_mask, segment_ids, label_ids = batch\n",
        "\n",
        "        # define a new function to compute loss values for both output_modes\n",
        "        logits = model(input_ids, segment_ids, input_mask, labels=None)\n",
        "        if mode == \"classification\":\n",
        "            loss_fct = CrossEntropyLoss()\n",
        "            loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1))\n",
        "        elif mode == \"regression\":\n",
        "            loss_fct = MSELoss()\n",
        "            loss = loss_fct(logits.view(-1), label_ids.view(-1))\n",
        "        loss.backward()\n",
        "\n",
        "        # Select maximum score index\n",
        "        _, preds = torch.max(logits, 1)\n",
        "\n",
        "        running_corrects += float(torch.sum(preds.data == label_ids.data))\n",
        "        tr_loss += loss.item()\n",
        "        nb_tr_examples += input_ids.size(0)\n",
        "\n",
        "        nb_tr_steps += 1\n",
        "        optimizer.step()\n",
        "        optimizer.zero_grad()\n",
        "        global_step += 1\n",
        "        if step%500 == 0:\n",
        "          print(\"Accuracy at step {}: {}\".format(step, running_corrects/nb_tr_examples))\n",
        "    epoch_acc = running_corrects.double().detach() / nb_tr_examples\n",
        "    epoch_acc = epoch_acc.data.cpu().numpy()\n",
        "    print(\"Epoch {}, accuracy: {}\".format(_, epoch_acc))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "Epoch:   0%|          | 0/4 [00:00<?, ?it/s]\u001b[A\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "451216\n",
            "Accuracy at step 0: 0.875\n",
            "Accuracy at step 500: 0.8670159680638723\n",
            "Accuracy at step 1000: 0.8921703296703297\n",
            "Accuracy at step 1500: 0.9009826782145236\n",
            "Accuracy at step 2000: 0.9047351324337831\n",
            "Accuracy at step 2500: 0.9079618152738904\n",
            "Accuracy at step 3000: 0.9111546151282905\n",
            "Accuracy at step 3500: 0.913560411311054\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XCTWtzJUp33a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}