nlp-suicidal-pytorch-bert-kfold.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.0209,
     "end_time": "2021-11-18T17:48:47.346967",
     "exception": false,
     "start_time": "2021-11-18T17:48:47.326067",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 1.Importing libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:01:30.971972Z",
     "iopub.status.busy": "2021-11-18T11:01:30.971581Z",
     "iopub.status.idle": "2021-11-18T11:01:36.614979Z",
     "shell.execute_reply": "2021-11-18T11:01:36.614122Z",
     "shell.execute_reply.started": "2021-11-18T11:01:30.971917Z"
    },
    "papermill": {
     "duration": 5.633518,
     "end_time": "2021-11-18T17:48:53.000338",
     "exception": false,
     "start_time": "2021-11-18T17:48:47.366820",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import pandas as pd \n",
    "import random \n",
    "import time\n",
    "import datetime\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, KFold\n",
    "from sklearn.metrics import precision_score, recall_score, f1_score\n",
    "import numpy as np \n",
    "from torch.utils.data import TensorDataset,Subset\n",
    "from transformers import BertTokenizer\n",
    "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
    "from torch.nn import functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "by5zkDazybkf",
    "papermill": {
     "duration": 0.030217,
     "end_time": "2021-11-18T17:48:53.071163",
     "exception": false,
     "start_time": "2021-11-18T17:48:53.040946",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 2.Enabling Gpu \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:01:46.803694Z",
     "iopub.status.busy": "2021-11-18T11:01:46.803316Z",
     "iopub.status.idle": "2021-11-18T11:01:46.83443Z",
     "shell.execute_reply": "2021-11-18T11:01:46.833428Z",
     "shell.execute_reply.started": "2021-11-18T11:01:46.803639Z"
    },
    "id": "oYsV4H8fCpZ-",
    "outputId": "9128471b-e9d2-4e22-a5ef-19b60bb45dee",
    "papermill": {
     "duration": 0.09265,
     "end_time": "2021-11-18T17:48:53.198295",
     "exception": false,
     "start_time": "2021-11-18T17:48:53.105645",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 1 GPU(s) available.\n",
      "We will use the GPU: Tesla P100-PCIE-16GB\n"
     ]
    }
   ],
   "source": [
    "# If there's a GPU available...\n",
    "if torch.cuda.is_available():    \n",
    "\n",
    "    # Tell PyTorch to use the GPU.    \n",
    "    device = torch.device(\"cuda\")\n",
    "\n",
    "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
    "\n",
    "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
    "\n",
    "# If not...\n",
    "else:\n",
    "    print('No GPU available, using the CPU instead.')\n",
    "    device = torch.device(\"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-8kEDRvShcU5",
    "papermill": {
     "duration": 0.032875,
     "end_time": "2021-11-18T17:48:53.266818",
     "exception": false,
     "start_time": "2021-11-18T17:48:53.233943",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 3.Organising Train Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:02:01.157881Z",
     "iopub.status.busy": "2021-11-18T11:02:01.157254Z",
     "iopub.status.idle": "2021-11-18T11:02:02.814525Z",
     "shell.execute_reply": "2021-11-18T11:02:02.813463Z",
     "shell.execute_reply.started": "2021-11-18T11:02:01.157637Z"
    },
    "id": "Z474sSC6oe7A",
    "outputId": "5e9fc033-2a6a-48e3-aac9-fed66b5d9e60",
    "papermill": {
     "duration": 0.777466,
     "end_time": "2021-11-18T17:48:54.077452",
     "exception": false,
     "start_time": "2021-11-18T17:48:53.299986",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30227e40d0ab4d899292cd72b9e20e9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=995526.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Load the BERT tokenizer.\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased', do_lower_case=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:02:08.455871Z",
     "iopub.status.busy": "2021-11-18T11:02:08.455518Z",
     "iopub.status.idle": "2021-11-18T11:02:08.484262Z",
     "shell.execute_reply": "2021-11-18T11:02:08.483585Z",
     "shell.execute_reply.started": "2021-11-18T11:02:08.455815Z"
    },
    "id": "yNU2Z5yk8iUn",
    "papermill": {
     "duration": 0.055728,
     "end_time": "2021-11-18T17:48:54.156139",
     "exception": false,
     "start_time": "2021-11-18T17:48:54.100411",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv('../input/datasetfinal/dataset_ideacao_final.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:02:12.917187Z",
     "iopub.status.busy": "2021-11-18T11:02:12.916834Z",
     "iopub.status.idle": "2021-11-18T11:02:12.951845Z",
     "shell.execute_reply": "2021-11-18T11:02:12.950548Z",
     "shell.execute_reply.started": "2021-11-18T11:02:12.917137Z"
    },
    "papermill": {
     "duration": 0.057376,
     "end_time": "2021-11-18T17:48:54.234783",
     "exception": false,
     "start_time": "2021-11-18T17:48:54.177407",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class preprocess_nlp(object):\n",
    "    \n",
    "    def __init__(self, texts, stopwords=False , lemma=False, stem=True, wordcloud=False):#, numeric='tfidf'):\n",
    "        \n",
    "        self.texts = texts\n",
    "        self.stopwords = stopwords\n",
    "        self.lemma = lemma\n",
    "        self.stem = stem\n",
    "        self.wordcloud = wordcloud\n",
    "        #self.numeric = numeric\n",
    "        self.new_texts = None\n",
    "        self.stopwords_list = list()\n",
    "        \n",
    "    def clean_text(self):\n",
    "\n",
    "        new_texts = list()\n",
    "\n",
    "        for text in self.texts:\n",
    "\n",
    "            text = text.lower()\n",
    "            text = re.sub('@[^\\s]+', '', text)\n",
    "            text = unidecode(text)\n",
    "            text = re.sub('<[^<]+?>','', text)\n",
    "            text = ''.join(c for c in text if not c.isdigit())\n",
    "            text = re.sub('((www\\.[^\\s]+)|(https?://[^\\s]+)|(http?://[^\\s]+))', '', text)\n",
    "            text = ''.join(c for c in text if c not in punctuation)\n",
    "            new_texts.append(text)\n",
    "        \n",
    "        self.new_texts = new_texts\n",
    "\n",
    "    def create_stopwords(self):\n",
    "        \n",
    "        stop_words = list(set(stopwords.words('portuguese') + list(STOP_WORDS)))\n",
    "        \n",
    "        for word in stop_words:\n",
    "\n",
    "            self.stopwords_list.append(unidecode(word))\n",
    "       \n",
    "    \n",
    "    def add_stopword(self, word):\n",
    "        \n",
    "        self.stopwords_list += [word]\n",
    "        \n",
    "\n",
    "    def remove_stopwords(self):\n",
    "\n",
    "        new_texts = list()\n",
    "\n",
    "        for text in self.new_texts:\n",
    "\n",
    "            new_text = ''\n",
    "\n",
    "            for word in word_tokenize(text):\n",
    "\n",
    "                if word.lower() not in self.stopwords_list:\n",
    "\n",
    "                    new_text += ' ' + word\n",
    "\n",
    "            new_texts.append(new_text)\n",
    "\n",
    "        self.new_texts = new_texts\n",
    "\n",
    "\n",
    "    def extract_lemma(self):\n",
    "        \n",
    "        nlp = spacy.load(\"pt\")\n",
    "        new_texts = list()\n",
    "\n",
    "        for text in self.texts:\n",
    "\n",
    "            new_text = ''\n",
    "\n",
    "            for word in nlp(text):\n",
    "\n",
    "                new_text += ' ' + word.lemma_\n",
    "\n",
    "            new_texts.append(new_text)\n",
    "        \n",
    "        self.new_texts = new_texts\n",
    "    \n",
    "\n",
    "    def extract_stem(self):\n",
    "\n",
    "        stemmer = nltk.stem.SnowballStemmer('portuguese')\n",
    "        new_texts = list()\n",
    "\n",
    "        for text in self.texts:\n",
    "\n",
    "            new_text = ''\n",
    "\n",
    "            for word in word_tokenize(text):\n",
    "\n",
    "                new_text += ' ' + stemmer.stem(word)\n",
    "\n",
    "            new_texts.append(new_text)\n",
    "\n",
    "        self.new_texts = new_texts\n",
    "    \n",
    "\n",
    "    def word_cloud(self):\n",
    "\n",
    "        all_words = ' '.join([text for text in self.new_texts])\n",
    "        word_cloud = WordCloud(width= 800, height= 500,\n",
    "                               max_font_size = 110, background_color=\"white\",\n",
    "                               collocations = False).generate(all_words)\n",
    "        plt.figure(figsize=(20,10))\n",
    "        plt.imshow(word_cloud, interpolation='bilinear')\n",
    "        plt.axis(\"off\")\n",
    "        plt.show()\n",
    "        wordcloud.to_file(\"wordcloud.png\")\n",
    "        \n",
    "\n",
    "    def countvectorizer(self):\n",
    "\n",
    "        vect = CountVectorizer()\n",
    "        text_vect = vect.fit_transform(self.new_texts)\n",
    "\n",
    "        return text_vect\n",
    "    \n",
    "\n",
    "    def tfidfvectorizer(self):\n",
    "\n",
    "        vect = TfidfVectorizer(max_features=50)\n",
    "        text_vect = vect.fit_transform(self.new_texts)\n",
    "\n",
    "        return text_vect\n",
    "    \n",
    "    \n",
    "    def preprocess(self):\n",
    "\n",
    "        self.clean_text()\n",
    "        \n",
    "        if self.stopwords == True:\n",
    "            self.create_stopwords()\n",
    "            self.remove_stopwords()\n",
    "            \n",
    "        #if self.lemma == True:\n",
    "            #self.extract_lemma()\n",
    "        \n",
    "        #if self.stem == True:\n",
    "            #self.extract_stem() \n",
    "        \n",
    "        #if self.wordcloud == True:\n",
    "            #self.word_cloud()\n",
    "        \n",
    "        #if self.numeric == 'tfidf':\n",
    "            #text_vect = self.tfidfvectorizer()\n",
    "        #elif self.numeric == 'count':\n",
    "            #text_vect = self.countvectorizer()\n",
    "        #else:\n",
    "            #print('metodo nao mapeado!')\n",
    "            #exit()\n",
    "            \n",
    "        return self.new_texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:02:21.559782Z",
     "iopub.status.busy": "2021-11-18T11:02:21.559407Z",
     "iopub.status.idle": "2021-11-18T11:02:23.583338Z",
     "shell.execute_reply": "2021-11-18T11:02:23.582466Z",
     "shell.execute_reply.started": "2021-11-18T11:02:21.559715Z"
    },
    "papermill": {
     "duration": 1.674336,
     "end_time": "2021-11-18T17:48:55.930407",
     "exception": false,
     "start_time": "2021-11-18T17:48:54.256071",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /usr/share/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "from string import punctuation\n",
    "from spacy.lang.pt.stop_words import STOP_WORDS\n",
    "import nltk\n",
    "from nltk import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "stoplist = stopwords.words('portuguese')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:02:28.725025Z",
     "iopub.status.busy": "2021-11-18T11:02:28.724632Z",
     "iopub.status.idle": "2021-11-18T11:02:28.978663Z",
     "shell.execute_reply": "2021-11-18T11:02:28.977941Z",
     "shell.execute_reply.started": "2021-11-18T11:02:28.724968Z"
    },
    "papermill": {
     "duration": 0.273833,
     "end_time": "2021-11-18T17:48:56.227337",
     "exception": false,
     "start_time": "2021-11-18T17:48:55.953504",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Pré processamento\n",
    "prepro = preprocess_nlp(data['text'])\n",
    "data['text'] = prepro.preprocess()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:02:33.630527Z",
     "iopub.status.busy": "2021-11-18T11:02:33.63018Z",
     "iopub.status.idle": "2021-11-18T11:02:33.649261Z",
     "shell.execute_reply": "2021-11-18T11:02:33.648579Z",
     "shell.execute_reply.started": "2021-11-18T11:02:33.630477Z"
    },
    "papermill": {
     "duration": 0.040946,
     "end_time": "2021-11-18T17:48:56.289402",
     "exception": false,
     "start_time": "2021-11-18T17:48:56.248456",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aquela vontade de acabar com a minha vida voltou</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>to triste e com vontade de acabar com a minha ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>corinthians ta querendo acabar com minha vida ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>alguem poderia por favor me dar um tiro a acab...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>taylor tu vai acabar com a minha vida mulher</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3783</th>\n",
       "      <td>ai tu quer me matar kkkkk</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3784</th>\n",
       "      <td>lembra que eu falei que hoje ia ser bom mais u...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3785</th>\n",
       "      <td>oi vou me matar em breve</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3786</th>\n",
       "      <td>vou tocar quarta pra eu n me matar pq nur</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3787</th>\n",
       "      <td>devo me matar agora ou depois</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3788 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  target\n",
       "0      aquela vontade de acabar com a minha vida voltou       1\n",
       "1     to triste e com vontade de acabar com a minha ...       1\n",
       "2     corinthians ta querendo acabar com minha vida ...       0\n",
       "3     alguem poderia por favor me dar um tiro a acab...       1\n",
       "4          taylor tu vai acabar com a minha vida mulher       0\n",
       "...                                                 ...     ...\n",
       "3783                          ai tu quer me matar kkkkk       0\n",
       "3784  lembra que eu falei que hoje ia ser bom mais u...       1\n",
       "3785                          oi vou me matar em breve        1\n",
       "3786          vou tocar quarta pra eu n me matar pq nur       0\n",
       "3787                      devo me matar agora ou depois       1\n",
       "\n",
       "[3788 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:02:41.132594Z",
     "iopub.status.busy": "2021-11-18T11:02:41.13225Z",
     "iopub.status.idle": "2021-11-18T11:02:41.142248Z",
     "shell.execute_reply": "2021-11-18T11:02:41.141324Z",
     "shell.execute_reply.started": "2021-11-18T11:02:41.132541Z"
    },
    "papermill": {
     "duration": 0.032216,
     "end_time": "2021-11-18T17:48:56.343796",
     "exception": false,
     "start_time": "2021-11-18T17:48:56.311580",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "train, test = train_test_split(data, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:02:47.586088Z",
     "iopub.status.busy": "2021-11-18T11:02:47.585675Z",
     "iopub.status.idle": "2021-11-18T11:02:47.592247Z",
     "shell.execute_reply": "2021-11-18T11:02:47.591502Z",
     "shell.execute_reply.started": "2021-11-18T11:02:47.586031Z"
    },
    "papermill": {
     "duration": 0.028048,
     "end_time": "2021-11-18T17:48:56.393189",
     "exception": false,
     "start_time": "2021-11-18T17:48:56.365141",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sentences = train.text.values\n",
    "labels = train.target.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:02:53.128553Z",
     "iopub.status.busy": "2021-11-18T11:02:53.12821Z",
     "iopub.status.idle": "2021-11-18T11:02:54.741169Z",
     "shell.execute_reply": "2021-11-18T11:02:54.740196Z",
     "shell.execute_reply.started": "2021-11-18T11:02:53.1285Z"
    },
    "id": "2bBdb3pt8LuQ",
    "outputId": "9e8d8f8e-6a20-4782-f7b2-e7a8be453b99",
    "papermill": {
     "duration": 1.67375,
     "end_time": "2021-11-18T17:48:58.088214",
     "exception": false,
     "start_time": "2021-11-18T17:48:56.414464",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original:  se eu nao chegar do medico a tempo de ver a champions eu vou me matar\n",
      "Token IDs: tensor([  101, 10126, 14444, 10132, 10133, 53319, 10149, 48091,   169, 12238,\n",
      "        10104, 16719,   169, 27745, 14444, 12556, 10138, 10911, 42655,   102,\n",
      "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
      "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
      "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
      "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
      "            0,     0,     0,     0])\n"
     ]
    }
   ],
   "source": [
    "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
    "input_ids = []\n",
    "attention_masks = []\n",
    "\n",
    "# For every sentence...\n",
    "for sent in sentences:\n",
    "  \n",
    "    encoded_dict = tokenizer.encode_plus(\n",
    "                        sent,                      # Sentence to encode.\n",
    "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
    "                        max_length = 64,           # Pad & truncate all sentences.\n",
    "                        pad_to_max_length = True,\n",
    "                        return_attention_mask = True,   # Construct attn. masks.\n",
    "                        return_tensors = 'pt',     # Return pytorch tensors.\n",
    "                   )\n",
    "    \n",
    "    # Add the encoded sentence to the list.    \n",
    "    input_ids.append(encoded_dict['input_ids'])\n",
    "    \n",
    "    # And its attention mask (simply differentiates padding from non-padding).\n",
    "    attention_masks.append(encoded_dict['attention_mask'])\n",
    "\n",
    "# Convert the lists into tensors.\n",
    "input_ids = torch.cat(input_ids, dim=0)\n",
    "attention_masks = torch.cat(attention_masks, dim=0)\n",
    "labels = torch.tensor(labels)\n",
    "\n",
    "# Print sentence 0, now as a list of IDs.\n",
    "print('Original: ', sentences[0])\n",
    "print('Token IDs:', input_ids[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:03:33.465903Z",
     "iopub.status.busy": "2021-11-18T11:03:33.465542Z",
     "iopub.status.idle": "2021-11-18T11:03:33.472481Z",
     "shell.execute_reply": "2021-11-18T11:03:33.471627Z",
     "shell.execute_reply.started": "2021-11-18T11:03:33.465849Z"
    },
    "id": "BfX6p9Y0ejAf",
    "papermill": {
     "duration": 0.031629,
     "end_time": "2021-11-18T17:48:58.144151",
     "exception": false,
     "start_time": "2021-11-18T17:48:58.112522",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "batch_size = 64 \n",
    "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
    "#helper function to get train and val data loaders for each fold \n",
    "def get_data_loaders(dataset,train_indexes,val_indexes):\n",
    "    train_tensor = Subset(dataset,train_indexes)\n",
    "    val_tensor = Subset(dataset,val_indexes)\n",
    "    train_dataloader = DataLoader(\n",
    "            train_tensor, \n",
    "            sampler = RandomSampler(train_tensor), \n",
    "            batch_size = batch_size\n",
    "        )\n",
    "\n",
    "    val_dataloader = DataLoader(\n",
    "            val_tensor, \n",
    "            sampler = SequentialSampler(val_tensor), \n",
    "            batch_size = batch_size \n",
    "        )\n",
    "    return train_dataloader,val_dataloader\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:03:36.628425Z",
     "iopub.status.busy": "2021-11-18T11:03:36.62809Z",
     "iopub.status.idle": "2021-11-18T11:03:36.634959Z",
     "shell.execute_reply": "2021-11-18T11:03:36.634097Z",
     "shell.execute_reply.started": "2021-11-18T11:03:36.628373Z"
    },
    "papermill": {
     "duration": 0.029146,
     "end_time": "2021-11-18T17:48:58.195426",
     "exception": false,
     "start_time": "2021-11-18T17:48:58.166280",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Combine the training inputs into a TensorDataset.\n",
    "dataset = TensorDataset(input_ids, attention_masks, labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6gNcyNpmv88D",
    "papermill": {
     "duration": 0.022139,
     "end_time": "2021-11-18T17:48:58.239862",
     "exception": false,
     "start_time": "2021-11-18T17:48:58.217723",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 4. Organising test data for predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:03:47.348451Z",
     "iopub.status.busy": "2021-11-18T11:03:47.348115Z",
     "iopub.status.idle": "2021-11-18T11:03:47.752468Z",
     "shell.execute_reply": "2021-11-18T11:03:47.751725Z",
     "shell.execute_reply.started": "2021-11-18T11:03:47.3484Z"
    },
    "id": "mAN0LZBOOPVh",
    "outputId": "43cc9400-8ff5-4816-82da-51f152348116",
    "papermill": {
     "duration": 0.425329,
     "end_time": "2021-11-18T17:48:58.687750",
     "exception": false,
     "start_time": "2021-11-18T17:48:58.262421",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#df = pd.read_csv(\"../input/nlp-getting-started/test.csv\")\n",
    "df = test\n",
    "sentences = df.text.values\n",
    "input_ids = []\n",
    "attention_masks = []\n",
    "for sent in sentences:\n",
    "\n",
    "    encoded_dict = tokenizer.encode_plus(\n",
    "                        sent,                      # Sentence to encode.\n",
    "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
    "                        max_length = 64,           # Pad & truncate all sentences.\n",
    "                        pad_to_max_length = True,\n",
    "                        return_attention_mask = True,   # Construct attn. masks.\n",
    "                        return_tensors = 'pt',     # Return pytorch tensors.\n",
    "                   )\n",
    "    \n",
    "    # Add the encoded sentence to the list.    \n",
    "    input_ids.append(encoded_dict['input_ids'])\n",
    "    # And its attention mask (simply differentiates padding from non-padding).\n",
    "    attention_masks.append(encoded_dict['attention_mask'])\n",
    "\n",
    "# Convert the lists into tensors.\n",
    "input_ids = torch.cat(input_ids, dim=0)\n",
    "attention_masks = torch.cat(attention_masks, dim=0)\n",
    "# Set the batch size.  \n",
    "batch_size = 16  \n",
    "# Create the DataLoader.\n",
    "prediction_data = TensorDataset(input_ids, attention_masks)\n",
    "prediction_sampler = SequentialSampler(prediction_data)\n",
    "prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RqfmWwUR_Sox",
    "papermill": {
     "duration": 0.021515,
     "end_time": "2021-11-18T17:48:58.732408",
     "exception": false,
     "start_time": "2021-11-18T17:48:58.710893",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 5. Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:04:15.337504Z",
     "iopub.status.busy": "2021-11-18T11:04:15.337155Z",
     "iopub.status.idle": "2021-11-18T11:04:15.343736Z",
     "shell.execute_reply": "2021-11-18T11:04:15.342592Z",
     "shell.execute_reply.started": "2021-11-18T11:04:15.337453Z"
    },
    "id": "QsOYopORzKXp",
    "papermill": {
     "duration": 0.029763,
     "end_time": "2021-11-18T17:48:58.784007",
     "exception": false,
     "start_time": "2021-11-18T17:48:58.754244",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
    "def get_bert_model():\n",
    "    model = BertForSequenceClassification.from_pretrained(\n",
    "      \"bert-base-multilingual-cased\", \n",
    "      num_labels = 2,           \n",
    "      output_attentions = False, \n",
    "      output_hidden_states = False, \n",
    "    )\n",
    "    # Tell pytorch to run this model on the GPU.\n",
    "    model.cuda()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:04:21.894561Z",
     "iopub.status.busy": "2021-11-18T11:04:21.894158Z",
     "iopub.status.idle": "2021-11-18T11:04:21.900113Z",
     "shell.execute_reply": "2021-11-18T11:04:21.898837Z",
     "shell.execute_reply.started": "2021-11-18T11:04:21.894502Z"
    },
    "id": "9cQNvaZ9bnyy",
    "papermill": {
     "duration": 0.029987,
     "end_time": "2021-11-18T17:48:58.835975",
     "exception": false,
     "start_time": "2021-11-18T17:48:58.805988",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "# Function to calculate the accuracy of our predictions vs labels\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:04:25.836327Z",
     "iopub.status.busy": "2021-11-18T11:04:25.835985Z",
     "iopub.status.idle": "2021-11-18T11:04:25.841176Z",
     "shell.execute_reply": "2021-11-18T11:04:25.840318Z",
     "shell.execute_reply.started": "2021-11-18T11:04:25.836277Z"
    },
    "id": "gpt6tR83keZD",
    "papermill": {
     "duration": 0.02946,
     "end_time": "2021-11-18T17:48:58.887188",
     "exception": false,
     "start_time": "2021-11-18T17:48:58.857728",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def format_time(elapsed):\n",
    "    '''\n",
    "    Takes a time in seconds and returns a string hh:mm:ss\n",
    "    '''\n",
    "    elapsed_rounded = int(round((elapsed)))\n",
    "    return str(datetime.timedelta(seconds=elapsed_rounded))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:04:29.669439Z",
     "iopub.status.busy": "2021-11-18T11:04:29.6691Z",
     "iopub.status.idle": "2021-11-18T11:04:29.674779Z",
     "shell.execute_reply": "2021-11-18T11:04:29.673567Z",
     "shell.execute_reply.started": "2021-11-18T11:04:29.669389Z"
    },
    "id": "LbB0miJTbLda",
    "papermill": {
     "duration": 0.029115,
     "end_time": "2021-11-18T17:48:58.938241",
     "exception": false,
     "start_time": "2021-11-18T17:48:58.909126",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set the seed value all over the place to make this reproducible.\n",
    "seed_val = 1000\n",
    "random.seed(seed_val)\n",
    "np.random.seed(seed_val)\n",
    "torch.manual_seed(seed_val)\n",
    "torch.cuda.manual_seed_all(seed_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:48:41.934056Z",
     "iopub.status.busy": "2021-11-18T11:48:41.93366Z",
     "iopub.status.idle": "2021-11-18T11:48:41.939745Z",
     "shell.execute_reply": "2021-11-18T11:48:41.938667Z",
     "shell.execute_reply.started": "2021-11-18T11:48:41.934Z"
    },
    "id": "khUH0dGRg4CM",
    "papermill": {
     "duration": 0.02968,
     "end_time": "2021-11-18T17:48:58.989903",
     "exception": false,
     "start_time": "2021-11-18T17:48:58.960223",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "total_folds = 5\n",
    "current_fold = 0\n",
    "epochs = 2\n",
    "all_folds_preds = []\n",
    "fold=StratifiedKFold(n_splits=total_folds, shuffle=True, random_state=1000)\n",
    "\n",
    "training_stats = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:48:51.795435Z",
     "iopub.status.busy": "2021-11-18T11:48:51.795086Z",
     "iopub.status.idle": "2021-11-18T11:53:22.709206Z",
     "shell.execute_reply": "2021-11-18T11:53:22.708448Z",
     "shell.execute_reply.started": "2021-11-18T11:48:51.795379Z"
    },
    "id": "6J-FYdx6nFE_",
    "outputId": "9f9bc3eb-4217-4379-82d3-0816826dad5b",
    "papermill": {
     "duration": 304.991054,
     "end_time": "2021-11-18T17:54:04.002948",
     "exception": false,
     "start_time": "2021-11-18T17:48:59.011894",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e44c842bb9a74c47aeab0eaf5b0c7cef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=625.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0c866cfb60f49a18c6b1dd70fef0515",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=714314041.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "================= Fold 1 / 5 =================\n",
      "\n",
      "======== Epoch 1 / 2 ========\n",
      "Training...\n",
      "\n",
      "  Average training loss: 0.35\n",
      "  Training epcoh took: 0:00:23\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.88\n",
      "  Precision_score: 0.89\n",
      "  Recall_score: 0.74\n",
      "  F1_score: 0.79\n",
      "  Validation Loss: 0.35\n",
      "  Validation took: 0:00:01\n",
      "\n",
      "======== Epoch 2 / 2 ========\n",
      "Training...\n",
      "\n",
      "  Average training loss: 0.24\n",
      "  Training epcoh took: 0:00:22\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.91\n",
      "  Precision_score: 0.79\n",
      "  Recall_score: 0.88\n",
      "  F1_score: 0.81\n",
      "  Validation Loss: 0.28\n",
      "  Validation took: 0:00:01\n",
      "\n",
      "Training complete!\n",
      "Total training took 0:01:28 (h:mm:ss)\n",
      "\n",
      "now predicting for this fold\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:190: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "================= Fold 2 / 5 =================\n",
      "\n",
      "======== Epoch 1 / 2 ========\n",
      "Training...\n",
      "\n",
      "  Average training loss: 0.36\n",
      "  Training epcoh took: 0:00:22\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.91\n",
      "  Precision_score: 0.81\n",
      "  Recall_score: 0.89\n",
      "  F1_score: 0.83\n",
      "  Validation Loss: 0.25\n",
      "  Validation took: 0:00:01\n",
      "\n",
      "======== Epoch 2 / 2 ========\n",
      "Training...\n",
      "\n",
      "  Average training loss: 0.23\n",
      "  Training epcoh took: 0:00:22\n",
      "\n",
      "Running Validation...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 due to no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Accuracy: 0.89\n",
      "  Precision_score: 0.66\n",
      "  Recall_score: 0.96\n",
      "  F1_score: 0.77\n",
      "  Validation Loss: 0.31\n",
      "  Validation took: 0:00:01\n",
      "\n",
      "Training complete!\n",
      "Total training took 0:02:22 (h:mm:ss)\n",
      "\n",
      "now predicting for this fold\n",
      "\n",
      "================= Fold 3 / 5 =================\n",
      "\n",
      "======== Epoch 1 / 2 ========\n",
      "Training...\n",
      "\n",
      "  Average training loss: 0.37\n",
      "  Training epcoh took: 0:00:22\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.88\n",
      "  Precision_score: 0.88\n",
      "  Recall_score: 0.74\n",
      "  F1_score: 0.79\n",
      "  Validation Loss: 0.33\n",
      "  Validation took: 0:00:01\n",
      "\n",
      "======== Epoch 2 / 2 ========\n",
      "Training...\n",
      "\n",
      "  Average training loss: 0.24\n",
      "  Training epcoh took: 0:00:22\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.92\n",
      "  Precision_score: 0.84\n",
      "  Recall_score: 0.85\n",
      "  F1_score: 0.83\n",
      "  Validation Loss: 0.27\n",
      "  Validation took: 0:00:01\n",
      "\n",
      "Training complete!\n",
      "Total training took 0:03:16 (h:mm:ss)\n",
      "\n",
      "now predicting for this fold\n",
      "\n",
      "================= Fold 4 / 5 =================\n",
      "\n",
      "======== Epoch 1 / 2 ========\n",
      "Training...\n",
      "\n",
      "  Average training loss: 0.40\n",
      "  Training epcoh took: 0:00:22\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.89\n",
      "  Precision_score: 0.91\n",
      "  Recall_score: 0.77\n",
      "  F1_score: 0.81\n",
      "  Validation Loss: 0.35\n",
      "  Validation took: 0:00:01\n",
      "\n",
      "======== Epoch 2 / 2 ========\n",
      "Training...\n",
      "\n",
      "  Average training loss: 0.25\n",
      "  Training epcoh took: 0:00:22\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.90\n",
      "  Precision_score: 0.87\n",
      "  Recall_score: 0.79\n",
      "  F1_score: 0.82\n",
      "  Validation Loss: 0.37\n",
      "  Validation took: 0:00:01\n",
      "\n",
      "Training complete!\n",
      "Total training took 0:04:09 (h:mm:ss)\n",
      "\n",
      "now predicting for this fold\n",
      "\n",
      "================= Fold 5 / 5 =================\n",
      "\n",
      "======== Epoch 1 / 2 ========\n",
      "Training...\n",
      "\n",
      "  Average training loss: 0.37\n",
      "  Training epcoh took: 0:00:22\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.90\n",
      "  Precision_score: 0.86\n",
      "  Recall_score: 0.82\n",
      "  F1_score: 0.83\n",
      "  Validation Loss: 0.27\n",
      "  Validation took: 0:00:01\n",
      "\n",
      "======== Epoch 2 / 2 ========\n",
      "Training...\n",
      "\n",
      "  Average training loss: 0.25\n",
      "  Training epcoh took: 0:00:23\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.90\n",
      "  Precision_score: 0.73\n",
      "  Recall_score: 0.93\n",
      "  F1_score: 0.80\n",
      "  Validation Loss: 0.26\n",
      "  Validation took: 0:00:01\n",
      "\n",
      "Training complete!\n",
      "Total training took 0:05:03 (h:mm:ss)\n",
      "\n",
      "now predicting for this fold\n",
      "Completed\n"
     ]
    }
   ],
   "source": [
    "# Measure the total training time for the whole run.\n",
    "total_t0 = time.time()\n",
    "#for each fold..\n",
    "for train_index, test_index in fold.split(train,train['target']):\n",
    "    model = get_bert_model()\n",
    "    optimizer = AdamW(model.parameters(),lr = 3e-5,eps = 1e-8)\n",
    "    current_fold = current_fold+1\n",
    "    train_dataloader,validation_dataloader = get_data_loaders(dataset,train_index,test_index)\n",
    "    print(\"\")\n",
    "    print('================= Fold {:} / {:} ================='.format(current_fold,total_folds))\n",
    "    # For each epoch...\n",
    "    for epoch_i in range(0, epochs):\n",
    "        # ========================================\n",
    "        #               Training\n",
    "        # ========================================\n",
    "\n",
    "        # Perform one full pass over the training set.\n",
    "\n",
    "        print(\"\")\n",
    "        print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
    "        print('Training...')\n",
    "\n",
    "        # Measure how long the training epoch takes.\n",
    "        t0 = time.time()\n",
    "\n",
    "        # Reset the total loss for this epoch.\n",
    "        total_train_loss = 0\n",
    "        model.train()\n",
    "        # For each batch of training data...\n",
    "        for step, batch in enumerate(train_dataloader):\n",
    "\n",
    "            b_input_ids = batch[0].to(device)\n",
    "            b_input_mask = batch[1].to(device)\n",
    "            b_labels = batch[2].to(device)\n",
    "            model.zero_grad()        \n",
    "\n",
    "            loss, logits = model(b_input_ids, \n",
    "                              token_type_ids=None, \n",
    "                              attention_mask=b_input_mask, \n",
    "                              labels=b_labels)\n",
    "\n",
    "\n",
    "            total_train_loss += loss.item()\n",
    "\n",
    "            # Perform a backward pass to calculate the gradients.\n",
    "            loss.backward()\n",
    "\n",
    "            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "            #update weights\n",
    "            optimizer.step()\n",
    "\n",
    "\n",
    "        # Calculate the average loss over all of the batches.\n",
    "        avg_train_loss = total_train_loss / len(train_dataloader)            \n",
    "\n",
    "        # Measure how long this epoch took.\n",
    "        training_time = format_time(time.time() - t0)\n",
    "\n",
    "        print(\"\")\n",
    "        print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
    "        print(\"  Training epcoh took: {:}\".format(training_time))\n",
    "\n",
    "        # ========================================\n",
    "        #               Validation\n",
    "        # ========================================\n",
    "        # After the completion of each training epoch, measure our performance on\n",
    "        # our validation set.\n",
    "\n",
    "        print(\"\")\n",
    "        print(\"Running Validation...\")\n",
    "\n",
    "        t0 = time.time()\n",
    "\n",
    "        # Put the model in evaluation mode--the dropout layers behave differently\n",
    "        # during evaluation.\n",
    "        model.eval()\n",
    "\n",
    "        # Tracking variables \n",
    "        total_f1_score = 0\n",
    "        total_eval_accuracy = 0\n",
    "        total_eval_loss = 0\n",
    "        nb_eval_steps = 0\n",
    "        total_precision_score = 0\n",
    "        total_recall_score = 0\n",
    "\n",
    "        # Evaluate data for one epoch\n",
    "        for batch in validation_dataloader:\n",
    "\n",
    "\n",
    "            b_input_ids = batch[0].to(device)\n",
    "            b_input_mask = batch[1].to(device)\n",
    "            b_labels = batch[2].to(device)\n",
    "\n",
    "            with torch.no_grad():        \n",
    "                (loss, logits) = model(b_input_ids, \n",
    "                                        token_type_ids=None, \n",
    "                                        attention_mask=b_input_mask,\n",
    "                                        labels=b_labels)\n",
    "\n",
    "            # Accumulate the validation loss.\n",
    "            total_eval_loss += loss.item()\n",
    "\n",
    "            # Move logits and labels to CPU\n",
    "            logits = logits.detach().cpu().numpy()\n",
    "            label_ids = b_labels.to('cpu').numpy()\n",
    "\n",
    "            # Calculate the accuracy for this batch of test sentences, and\n",
    "            # accumulate it over all batches.\n",
    "            total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
    "            total_f1_score += f1_score(np.argmax(logits,axis=1),label_ids)\n",
    "            total_precision_score += precision_score(np.argmax(logits,axis=1),label_ids)\n",
    "            total_recall_score += recall_score(np.argmax(logits,axis=1),label_ids)\n",
    "\n",
    "        # Report the final accuracy and f1_score for this validation run.\n",
    "        avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
    "        print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
    "        \n",
    "        avg_precision_score = total_precision_score / len(validation_dataloader)\n",
    "        print(\"  Precision_score: {0:.2f}\".format(avg_precision_score))\n",
    "        \n",
    "        avg_recall_score = total_recall_score / len(validation_dataloader)\n",
    "        print(\"  Recall_score: {0:.2f}\".format(avg_recall_score))\n",
    "        \n",
    "        avg_f1_score = total_f1_score / len(validation_dataloader)\n",
    "        print(\"  F1_score: {0:.2f}\".format(avg_f1_score))\n",
    "\n",
    "        # Calculate the average loss over all of the batches.\n",
    "        avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
    "\n",
    "        # Measure how long the validation run took.\n",
    "        validation_time = format_time(time.time() - t0)\n",
    "\n",
    "        print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
    "        print(\"  Validation took: {:}\".format(validation_time))\n",
    "\n",
    "        # Record all statistics from this epoch.\n",
    "        training_stats.append(\n",
    "          {\n",
    "              'epoch': epoch_i + 1,\n",
    "              'Training Loss': avg_train_loss,\n",
    "              'Valid. Loss': avg_val_loss,\n",
    "              'Valid. Accur.': avg_val_accuracy,\n",
    "              'Precision_score': avg_precision_score,\n",
    "              'Recall_score': avg_recall_score,\n",
    "              'f1_score' : avg_f1_score,\n",
    "              'Training Time': training_time,\n",
    "              'Validation Time': validation_time,\n",
    "              'fold' : current_fold\n",
    "              \n",
    "          }\n",
    "        )\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Training complete!\")\n",
    "\n",
    "    print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))\n",
    "\n",
    "    # ========================================\n",
    "    # Predicting and saving predictions for all folds\n",
    "    # ========================================\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"now predicting for this fold\")\n",
    "\n",
    "    # Put model in evaluation mode\n",
    "    model.eval()\n",
    "    # Tracking variables \n",
    "    predictions  = []\n",
    "    # Predict \n",
    "    for batch in prediction_dataloader:\n",
    "        # Add batch to GPU\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        # Unpack the inputs from our dataloader\n",
    "        b_input_ids, b_input_mask = batch\n",
    "        # speeding up prediction\n",
    "        with torch.no_grad():\n",
    "            # Forward pass, calculate logit predictions\n",
    "            outputs = model(b_input_ids, token_type_ids=None, \n",
    "                            attention_mask=b_input_mask)\n",
    "\n",
    "        logits = outputs[0]\n",
    "\n",
    "        # Move logits and labels to CPU\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "\n",
    "\n",
    "        predictions.append(logits)\n",
    "\n",
    "    stack = np.vstack(predictions)\n",
    "    final_preds = F.softmax(torch.from_numpy(stack))[:,1].numpy()\n",
    "    all_folds_preds.append(final_preds)\n",
    "print(\"Completed\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-18T11:47:40.056998Z",
     "iopub.status.busy": "2021-11-18T11:47:40.056603Z",
     "iopub.status.idle": "2021-11-18T11:47:40.079876Z",
     "shell.execute_reply": "2021-11-18T11:47:40.078886Z",
     "shell.execute_reply.started": "2021-11-18T11:47:40.056939Z"
    },
    "id": "6O_NbXFGMukX",
    "papermill": {
     "duration": 0.056088,
     "end_time": "2021-11-18T17:54:04.092927",
     "exception": false,
     "start_time": "2021-11-18T17:54:04.036839",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Valid. Loss</th>\n",
       "      <th>Valid. Accur.</th>\n",
       "      <th>Precision_score</th>\n",
       "      <th>Recall_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>Training Time</th>\n",
       "      <th>Validation Time</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fold</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.89</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.79</td>\n",
       "      <td>0:00:23</td>\n",
       "      <td>0:00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.79</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.81</td>\n",
       "      <td>0:00:22</td>\n",
       "      <td>0:00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.81</td>\n",
       "      <td>0.89</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0:00:22</td>\n",
       "      <td>0:00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.89</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.96</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0:00:22</td>\n",
       "      <td>0:00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0.37</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.79</td>\n",
       "      <td>0:00:22</td>\n",
       "      <td>0:00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.92</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0:00:22</td>\n",
       "      <td>0:00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.89</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.81</td>\n",
       "      <td>0:00:22</td>\n",
       "      <td>0:00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.37</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.79</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0:00:22</td>\n",
       "      <td>0:00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>0.37</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0:00:22</td>\n",
       "      <td>0:00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.26</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0:00:23</td>\n",
       "      <td>0:00:01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      epoch  Training Loss  Valid. Loss  Valid. Accur.  Precision_score  \\\n",
       "fold                                                                      \n",
       "1         1           0.35         0.35           0.88             0.89   \n",
       "1         2           0.24         0.28           0.91             0.79   \n",
       "2         1           0.36         0.25           0.91             0.81   \n",
       "2         2           0.23         0.31           0.89             0.66   \n",
       "3         1           0.37         0.33           0.88             0.88   \n",
       "3         2           0.24         0.27           0.92             0.84   \n",
       "4         1           0.40         0.35           0.89             0.91   \n",
       "4         2           0.25         0.37           0.90             0.87   \n",
       "5         1           0.37         0.27           0.90             0.86   \n",
       "5         2           0.25         0.26           0.90             0.73   \n",
       "\n",
       "      Recall_score  f1_score Training Time Validation Time  \n",
       "fold                                                        \n",
       "1             0.74      0.79       0:00:23         0:00:01  \n",
       "1             0.88      0.81       0:00:22         0:00:01  \n",
       "2             0.89      0.83       0:00:22         0:00:01  \n",
       "2             0.96      0.77       0:00:22         0:00:01  \n",
       "3             0.74      0.79       0:00:22         0:00:01  \n",
       "3             0.85      0.83       0:00:22         0:00:01  \n",
       "4             0.77      0.81       0:00:22         0:00:01  \n",
       "4             0.79      0.82       0:00:22         0:00:01  \n",
       "5             0.82      0.83       0:00:22         0:00:01  \n",
       "5             0.93      0.80       0:00:23         0:00:01  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('precision', 2)\n",
    "df_stats = pd.DataFrame(data=training_stats)\n",
    "df_stats = df_stats.set_index('fold')\n",
    "df_stats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mkyubuJSOzg3",
    "papermill": {
     "duration": 0.03131,
     "end_time": "2021-11-18T17:54:04.156490",
     "exception": false,
     "start_time": "2021-11-18T17:54:04.125180",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 6. Setting File Submission\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-12T19:07:43.011931Z",
     "iopub.status.busy": "2021-09-12T19:07:43.011573Z",
     "iopub.status.idle": "2021-09-12T19:07:43.016322Z",
     "shell.execute_reply": "2021-09-12T19:07:43.015178Z",
     "shell.execute_reply.started": "2021-09-12T19:07:43.011878Z"
    },
    "id": "3bWYxgST3X8k",
    "papermill": {
     "duration": 0.037917,
     "end_time": "2021-11-18T17:54:04.226396",
     "exception": false,
     "start_time": "2021-11-18T17:54:04.188479",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "to_submit =np.mean(all_folds_preds,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-12T19:07:56.963135Z",
     "iopub.status.busy": "2021-09-12T19:07:56.962751Z",
     "iopub.status.idle": "2021-09-12T19:07:56.983991Z",
     "shell.execute_reply": "2021-09-12T19:07:56.982781Z",
     "shell.execute_reply.started": "2021-09-12T19:07:56.963085Z"
    },
    "id": "844CyjlFjwfw",
    "outputId": "16b63c8f-2cda-4b80-dcc4-4e221a0e6c7b",
    "papermill": {
     "duration": 0.052317,
     "end_time": "2021-11-18T17:54:04.310772",
     "exception": false,
     "start_time": "2021-11-18T17:54:04.258455",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>185</th>\n",
       "      <td>para de fingir q ta melhor sem mim besta e dei...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2787</th>\n",
       "      <td>quer me matar</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1424</th>\n",
       "      <td>quero morrer</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1932</th>\n",
       "      <td>ta querendo de tristeza me matar a essa hora</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2817</th>\n",
       "      <td>so um video do maicon e do kotaka pra me alegr...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  target\n",
       "185   para de fingir q ta melhor sem mim besta e dei...       0\n",
       "2787                                      quer me matar       0\n",
       "1424                                       quero morrer       1\n",
       "1932      ta querendo de tristeza me matar a essa hora        0\n",
       "2817  so um video do maicon e do kotaka pra me alegr...       0"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub=pd.DataFrame()\n",
    "sub['text'] = df['text']\n",
    "sub['target'] = to_submit\n",
    "sub['target'] = sub['target'].apply(lambda x: 1 if x>0.5 else 0)\n",
    "sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "b7SeQk7Sl4YZ",
    "papermill": {
     "duration": 0.085682,
     "end_time": "2021-11-18T17:54:04.428916",
     "exception": false,
     "start_time": "2021-11-18T17:54:04.343234",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sub.to_csv('bert_base_12_2e-5_64.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.033164,
     "end_time": "2021-11-18T17:54:04.495693",
     "exception": false,
     "start_time": "2021-11-18T17:54:04.462529",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "papermill": {
   "duration": 322.068419,
   "end_time": "2021-11-18T17:54:05.938684",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-11-18T17:48:43.870265",
   "version": "1.2.1"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "128d314e5ef0474fa98422bc766552d0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "14e0b967e80f4842b82be0037a836fb6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "1a03e17c315c49619e29c9da62f65a34": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "22a9841c8f024a4996fef4914fc7a861": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "2b4f42bd810c48f8ba943bff5a32afa7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "30227e40d0ab4d899292cd72b9e20e9b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_7958f3c26e6446fab8c54e416be17326",
        "IPY_MODEL_869ca3cc48fa46a99313a517cbec5cf2"
       ],
       "layout": "IPY_MODEL_4bcbf64cd34a4ec5a7b7e8c877d18b63"
      }
     },
     "3bbb3790a75b40fabca06952af0f0c67": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_673e83ea2b814e95ae984408a6126e5d",
       "max": 625.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_2b4f42bd810c48f8ba943bff5a32afa7",
       "value": 625.0
      }
     },
     "3e30e2011a90422a9b3a88be3c008531": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_b151f4697b4540d59f3d744750523299",
       "max": 714314041.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_128d314e5ef0474fa98422bc766552d0",
       "value": 714314041.0
      }
     },
     "4bcbf64cd34a4ec5a7b7e8c877d18b63": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "53c3d6cae8974aeb9799e1e4a5b564ce": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_611ffccfb7e049fb9b85aa2d4b0f3e18",
       "placeholder": "​",
       "style": "IPY_MODEL_22a9841c8f024a4996fef4914fc7a861",
       "value": " 625/625 [00:00&lt;00:00, 21.7kB/s]"
      }
     },
     "597ba2eea2e84d11ad05d7b380fc1b38": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_d9b2bd82b4044a6b965a30a111353993",
       "placeholder": "​",
       "style": "IPY_MODEL_14e0b967e80f4842b82be0037a836fb6",
       "value": " 714M/714M [00:30&lt;00:00, 23.5MB/s]"
      }
     },
     "611ffccfb7e049fb9b85aa2d4b0f3e18": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "673e83ea2b814e95ae984408a6126e5d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7958f3c26e6446fab8c54e416be17326": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_1a03e17c315c49619e29c9da62f65a34",
       "max": 995526.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_ba8be4e3b7434c9fa285ada194a4e913",
       "value": 995526.0
      }
     },
     "821c93dfbed24cdb965e65dead190b3b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "869ca3cc48fa46a99313a517cbec5cf2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_b689390fc2f744ae9a5ff6f2765fddfe",
       "placeholder": "​",
       "style": "IPY_MODEL_c9ba4a032632498e8cbdda7307134d1c",
       "value": " 996k/996k [00:00&lt;00:00, 3.60MB/s]"
      }
     },
     "b151f4697b4540d59f3d744750523299": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b689390fc2f744ae9a5ff6f2765fddfe": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ba8be4e3b7434c9fa285ada194a4e913": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "bb5847c82d2b43dd8da36786bfc50104": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c9ba4a032632498e8cbdda7307134d1c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "d9b2bd82b4044a6b965a30a111353993": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e0c866cfb60f49a18c6b1dd70fef0515": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_3e30e2011a90422a9b3a88be3c008531",
        "IPY_MODEL_597ba2eea2e84d11ad05d7b380fc1b38"
       ],
       "layout": "IPY_MODEL_bb5847c82d2b43dd8da36786bfc50104"
      }
     },
     "e44c842bb9a74c47aeab0eaf5b0c7cef": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_3bbb3790a75b40fabca06952af0f0c67",
        "IPY_MODEL_53c3d6cae8974aeb9799e1e4a5b564ce"
       ],
       "layout": "IPY_MODEL_821c93dfbed24cdb965e65dead190b3b"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}