T5_V0.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "T5-V0.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyP4foqK3OEHiDgYJfG/lX0I",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "6b9853a1bad541d797f23c9e7f626afa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_78f2d53ccb5942e9bf009effcd7415e7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_70824ed2807a49f098d8debd6b458dcf",
              "IPY_MODEL_cfb61b4d751e49ad978cda2152919703"
            ]
          }
        },
        "78f2d53ccb5942e9bf009effcd7415e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "70824ed2807a49f098d8debd6b458dcf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_32ce913fddc24f438c1090602e2ba8c0",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 791656,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791656,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_39b3c3ab3e9b4da2b1ec1f6ef126feeb"
          }
        },
        "cfb61b4d751e49ad978cda2152919703": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_50b461e63cd24befae99ceccefc527f3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 792k/792k [00:05&lt;00:00, 155kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bcf36ff2caea439796056609837b271c"
          }
        },
        "32ce913fddc24f438c1090602e2ba8c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "39b3c3ab3e9b4da2b1ec1f6ef126feeb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "50b461e63cd24befae99ceccefc527f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bcf36ff2caea439796056609837b271c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/finardi/Fast-Marching-master-degree-/blob/master/T5_V0.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PcMfqlK-0HLY",
        "colab_type": "code",
        "outputId": "b9b19863-cb4e-48f4-f086-4f9255a8933f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 326
        }
      },
      "source": [
        "! nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mon May 25 13:06:27 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 440.82       Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   34C    P0    25W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                       GPU Memory |\n",
            "|  GPU       PID   Type   Process name                             Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c-1Lmkfv6g2j",
        "colab_type": "code",
        "outputId": "4a3524bb-439d-4764-8911-bab405919358",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        }
      },
      "source": [
        "! rm -rf date*\n",
        "! git clone https://github.com/textnorms/date_text_norm.git\n",
        "! cp -r date_text_norm/syntetic_data/ .\n",
        "! pip install -q transformers\n",
        "! pip install -q num2words"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'date_text_norm'...\n",
            "remote: Enumerating objects: 43, done.\u001b[K\n",
            "remote: Counting objects: 100% (43/43), done.\u001b[K\n",
            "remote: Compressing objects: 100% (30/30), done.\u001b[K\n",
            "remote: Total 43 (delta 16), reused 29 (delta 8), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (43/43), done.\n",
            "\u001b[K     |████████████████████████████████| 665kB 2.8MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.8MB 10.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1MB 31.9MB/s \n",
            "\u001b[K     |████████████████████████████████| 890kB 51.1MB/s \n",
            "\u001b[?25h  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[K     |████████████████████████████████| 102kB 2.3MB/s \n",
            "\u001b[?25h"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wuqR9pkb4XxT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nok2mtt_1021",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Basics\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import random\n",
        "\n",
        "# Num2words and dates\n",
        "from num2words import num2words\n",
        "from datetime import datetime, timedelta\n",
        "\n",
        "# PyTorch\n",
        "import torch \n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "\n",
        "# Sklearn\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Transformers\n",
        "from transformers import T5Tokenizer, T5ForConditionalGeneration, AdamW"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HhdTvVgiYBw0",
        "colab_type": "text"
      },
      "source": [
        "### Função para reproduzir resultados"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zm7tAsiUYMOU",
        "colab_type": "code",
        "outputId": "aea23c62-f12c-41e8-e589-bb15b4e390bd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "manual_seed = 0\n",
        "def deterministic(rep=True):\n",
        "    if rep:\n",
        "        np.random.seed(manual_seed)\n",
        "        torch.manual_seed(manual_seed)\n",
        "        if torch.cuda.is_available():\n",
        "            torch.cuda.manual_seed(manual_seed)\n",
        "            torch.cuda.manual_seed_all(manual_seed)\n",
        "        torch.backends.cudnn.enabled = False \n",
        "        torch.backends.cudnn.benchmark = False\n",
        "        torch.backends.cudnn.deterministic = True\n",
        "        print(f'Deterministic experiment, seed: {manual_seed}')\n",
        "    else:\n",
        "        print('Random experiment')\n",
        "\n",
        "deterministic()\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "print(f'Using device: {device}')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Deterministic experiment, seed: 0\n",
            "Using device: cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1yeoFiMYXe8Z",
        "colab_type": "text"
      },
      "source": [
        "# Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L9p0tFUkXiE8",
        "colab_type": "code",
        "outputId": "22bac059-9192-4102-b087-e669c5f77dcc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "6b9853a1bad541d797f23c9e7f626afa",
            "78f2d53ccb5942e9bf009effcd7415e7",
            "70824ed2807a49f098d8debd6b458dcf",
            "cfb61b4d751e49ad978cda2152919703",
            "32ce913fddc24f438c1090602e2ba8c0",
            "39b3c3ab3e9b4da2b1ec1f6ef126feeb",
            "50b461e63cd24befae99ceccefc527f3",
            "bcf36ff2caea439796056609837b271c"
          ]
        }
      },
      "source": [
        "model_size = 't5-small'\n",
        "tokenizer = T5Tokenizer.from_pretrained(model_size)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6b9853a1bad541d797f23c9e7f626afa",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=791656.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QAJYuuarstv_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "'''\n",
        "    Este arquivo contêm dicionários auxiliares para a construção\n",
        "    de datas por extenso em PT_BR. \n",
        "\n",
        "'''\n",
        "\n",
        "'''\n",
        "    Meses escritos por extenso\n",
        "'''\n",
        "extensive_months_dict = {\n",
        "    '01': 'janeiro',\n",
        "    '02': 'fevereiro',\n",
        "    '03': 'março',\n",
        "    '04': 'abril',\n",
        "    '05': 'maio',\n",
        "    '06': 'junho',\n",
        "    '07': 'julho',\n",
        "    '08': 'agosto',\n",
        "    '09': 'setembro',\n",
        "    '10': 'outubro',\n",
        "    '11': 'novembro',\n",
        "    '12': 'dezembro'\n",
        "}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9VG5SJ44HNEm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class DateTextGenerator():\n",
        "\n",
        "    '''\n",
        "    Essa classe implementa um gerador de texto sintético\n",
        "    que usa como entrada datas no formato canônico e produz\n",
        "    amostras em formatos textuais não canônicos. \n",
        "    E.g.:\n",
        "        - Entrada: 01/05/2020\n",
        "        - Saídas possíveis:\n",
        "            - 01 de maio de 2020;\n",
        "            - primeiro de maior de 2020;\n",
        "            - primeiro de maio de dois mil e vinte;\n",
        "            - primeiro do 05 de 2020;\n",
        "                .\n",
        "                .\n",
        "                .\n",
        "    '''\n",
        "    def __init__(self,start_date='01/01/0001',end_date='31/12/2999'):\n",
        "\n",
        "        self.start_date = datetime.strptime(start_date, \"%d/%m/%Y\")\n",
        "        self.end_date = datetime.strptime(end_date, \"%d/%m/%Y\")\n",
        "\n",
        "        self.date_range = self.generate_date_range(self.start_date,self.end_date)\n",
        "\n",
        "\n",
        "    def generate_date_dataset(self):\n",
        "\n",
        "        X = []\n",
        "\n",
        "        for sample in self.date_range:\n",
        "            day,month,year = sample.split('/')\n",
        "\n",
        "            X.append(\n",
        "                self._text_gen(day,month,year)\n",
        "            )\n",
        "\n",
        "        for sample in self.date_range:\n",
        "            day,month,year = sample.split('/')\n",
        "\n",
        "            X.append(\n",
        "                self._dot_as_sep(day,month,year)\n",
        "            )\n",
        "\n",
        "        for sample in self.date_range:\n",
        "            day,month,year = sample.split('/')\n",
        "\n",
        "            X.append(\n",
        "                self._all_extensive_numbers(day,month,year)\n",
        "            )\n",
        "\n",
        "        dataset = pd.DataFrame(list(zip(X,3*self.date_range)),columns=['inputs','labels'])\n",
        "\n",
        "        return dataset\n",
        "\n",
        "    @staticmethod\n",
        "    def _all_extensive_numbers(day,month,year):\n",
        "\n",
        "        input_day = num2words(int(day),lang='pt_BR')\n",
        "        input_month = num2words(int(month),lang='pt_BR')\n",
        "        input_year = num2words(int(year),lang='pt_BR')\n",
        "\n",
        "        return f'{input_day} do {input_month} de {input_year}'\n",
        "\n",
        "    @staticmethod\n",
        "    def _dot_as_sep(day,month,year):\n",
        "        return f'{day}.{month}.{year}'\n",
        "\n",
        "    @staticmethod\n",
        "    def _text_gen(day,month,year):\n",
        "\n",
        "        input_day = num2words(int(day),lang='pt_BR')\n",
        "        input_month = extensive_months_dict[month]\n",
        "        input_year = num2words(int(year),lang='pt_BR')\n",
        "\n",
        "        return f'{input_day} de {input_month} de {input_year}'\n",
        "\n",
        "    @staticmethod\n",
        "    def generate_date_range (start_date,end_date,step=1):\n",
        "        '''\n",
        "           Implementa um range de datas com os dias que estão entre\n",
        "           start_date e end_date. Implementação inspirada em:\n",
        "            https://gist.github.com/ramhiser/989263a7a136601e3723\n",
        "            e\n",
        "            https://stackoverflow.com/questions/339007/how-to-pad-zeroes-to-a-string\n",
        "        '''\n",
        "        \n",
        "        dates = []\n",
        "\n",
        "        for d in range(0, (end_date - start_date).days + step, step):\n",
        "            date_i = start_date + timedelta(days=d)\n",
        "            \n",
        "            dia = str(date_i.date().day).zfill(2)\n",
        "            mes = str(date_i.date().month).zfill(2)\n",
        "            ano = str(date_i.date().year).zfill(4)\n",
        "\n",
        "            dates.append(f'{dia}/{mes}/{ano}')\n",
        "\n",
        "        return dates"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eZ52EJP6Fy5v",
        "colab_type": "code",
        "outputId": "80170e87-c335-4ed0-a32d-3b595990e56c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 407
        }
      },
      "source": [
        "datas = DateTextGenerator(start_date='01/01/1900',end_date='31/12/2020')\n",
        "df = datas.generate_date_dataset(); df"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>inputs</th>\n",
              "      <th>labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>um de janeiro de mil, novecentos</td>\n",
              "      <td>01/01/1900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>dois de janeiro de mil, novecentos</td>\n",
              "      <td>02/01/1900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>três de janeiro de mil, novecentos</td>\n",
              "      <td>03/01/1900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>quatro de janeiro de mil, novecentos</td>\n",
              "      <td>04/01/1900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>cinco de janeiro de mil, novecentos</td>\n",
              "      <td>05/01/1900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>132580</th>\n",
              "      <td>vinte e sete do doze de dois mil e vinte</td>\n",
              "      <td>27/12/2020</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>132581</th>\n",
              "      <td>vinte e oito do doze de dois mil e vinte</td>\n",
              "      <td>28/12/2020</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>132582</th>\n",
              "      <td>vinte e nove do doze de dois mil e vinte</td>\n",
              "      <td>29/12/2020</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>132583</th>\n",
              "      <td>trinta do doze de dois mil e vinte</td>\n",
              "      <td>30/12/2020</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>132584</th>\n",
              "      <td>trinta e um do doze de dois mil e vinte</td>\n",
              "      <td>31/12/2020</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>132585 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                          inputs      labels\n",
              "0               um de janeiro de mil, novecentos  01/01/1900\n",
              "1             dois de janeiro de mil, novecentos  02/01/1900\n",
              "2             três de janeiro de mil, novecentos  03/01/1900\n",
              "3           quatro de janeiro de mil, novecentos  04/01/1900\n",
              "4            cinco de janeiro de mil, novecentos  05/01/1900\n",
              "...                                          ...         ...\n",
              "132580  vinte e sete do doze de dois mil e vinte  27/12/2020\n",
              "132581  vinte e oito do doze de dois mil e vinte  28/12/2020\n",
              "132582  vinte e nove do doze de dois mil e vinte  29/12/2020\n",
              "132583        trinta do doze de dois mil e vinte  30/12/2020\n",
              "132584   trinta e um do doze de dois mil e vinte  31/12/2020\n",
              "\n",
              "[132585 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FcgUZ2SI1uk2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# incluir exemplo\n",
        "# datas incompletas 1/2020 -> 01/2020 , 5/2020 -> 05/2020\n",
        "# 1/1/2010 -> 01/01/2010\n",
        "# aos cinco primeiros dias de...\n",
        "# data com erro de ocr: vint sete doze dois mil e vinte\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6WJfgr1VblzT",
        "colab_type": "code",
        "outputId": "f8574119-feb3-4e6a-9340-e0745e5df7cf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# split the data \n",
        "x_train, x_val, y_train, y_val = train_test_split(\n",
        "        df.inputs.values,\n",
        "        df.labels.values,\n",
        "        shuffle=True, \n",
        "        test_size=0.3, \n",
        "        random_state=manual_seed\n",
        "        )\n",
        "\n",
        "len(x_train), len(y_train), len(x_val), len(y_val)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(92809, 92809, 39776, 39776)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ygsMXnkCyrVZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# -------------------\n",
        "max_len_source = 40\n",
        "max_len_target = 12\n",
        "# -------------------\n",
        "\n",
        "class DateDataset(Dataset):\n",
        "    def __init__(self, data, label, tokenizer, source_max_length, target_max_length):\n",
        "        self.tokenizer = tokenizer\n",
        "        self.data = data\n",
        "        self.label = label\n",
        "        self.source_max_length = source_max_length\n",
        "        self.target_max_length = target_max_length\n",
        "        \n",
        "    def __len__(self):\n",
        "        return len(self.data)\n",
        "    \n",
        "    def __getitem__(self, idx):\n",
        "        source = self.data[idx]\n",
        "        target = self.label[idx]\n",
        "\n",
        "        source_tokenized = self.tokenizer.encode_plus(\n",
        "            f'{source} {self.tokenizer.eos_token}',\n",
        "            max_length=self.source_max_length,\n",
        "            pad_to_max_length=True,\n",
        "            return_tensors='pt')\n",
        "\n",
        "        target_tokenized = self.tokenizer.encode_plus(\n",
        "            f'{target} {self.tokenizer.eos_token}',\n",
        "            max_length=self.target_max_length,\n",
        "            pad_to_max_length=True,\n",
        "            return_tensors='pt')\n",
        "\n",
        "        source_token_ids = source_tokenized['input_ids'].squeeze()\n",
        "        source_mask = source_tokenized['attention_mask'].squeeze()\n",
        "        target_token_ids = target_tokenized['input_ids'].squeeze()\n",
        "        \n",
        "        return source_token_ids, source_mask, target_token_ids"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cloyt0tIwIiD",
        "colab_type": "text"
      },
      "source": [
        "## Teste da classe"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZoKiQXCvwGrP",
        "colab_type": "code",
        "outputId": "07011ec9-c3b0-46ae-e1ab-a773bc89f973",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 217
        }
      },
      "source": [
        "dataset_debug = DateDataset(\n",
        "    x_train, \n",
        "    y_train,\n",
        "    tokenizer,\n",
        "    max_len_source,\n",
        "    max_len_target,\n",
        "    )\n",
        "\n",
        "dataloader_debug = DataLoader(\n",
        "    dataset_debug, \n",
        "    batch_size=1, \n",
        "    shuffle=True, \n",
        "    num_workers=0\n",
        "    )\n",
        "\n",
        "source_token_ids, source_mask, target_token_ids = next(iter(dataloader_debug))\n",
        "print(f'source_token_ids:\\n {source_token_ids} --- shape:{source_token_ids.shape}')\n",
        "print(f'source_mask:\\n {source_mask} --- shape:{source_mask.shape}')\n",
        "print(f'target_token_ids:\\n {target_token_ids} --- shape:{target_token_ids.shape}')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "source_token_ids:\n",
            " tensor([[    3,   208,  2429,     3,    15,   561,    20,    91,    76,  5702,\n",
            "            20, 15533,     6,   150,   162,  3728,    32,     7,     3,    15,\n",
            "             3,    32,   155,   295,     9,     3,    15,   356,    15,     1,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0]]) --- shape:torch.Size([1, 40])\n",
            "source_mask:\n",
            " tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
            "         1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]) --- shape:torch.Size([1, 40])\n",
            "target_token_ids:\n",
            " tensor([[ 1401, 11476, 13523,  4225,     1,     0,     0,     0,     0,     0,\n",
            "             0,     0]]) --- shape:torch.Size([1, 12])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dG1O12UoWYaM",
        "colab_type": "text"
      },
      "source": [
        "## Datasets e Dataloaders"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s7AlRyeOW8GN",
        "colab_type": "code",
        "outputId": "2401c5f2-be73-4fba-9e41-4297822a479c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        }
      },
      "source": [
        "BATCH_SZ = 128\n",
        "\n",
        "# datasets\n",
        "ds_debug = DateDataset(\n",
        "    x_train[:BATCH_SZ], \n",
        "    y_train[:BATCH_SZ],\n",
        "    tokenizer,\n",
        "    max_len_source,\n",
        "    max_len_target\n",
        "    )\n",
        "\n",
        "ds_train = DateDataset(\n",
        "    x_train, \n",
        "    y_train,\n",
        "    tokenizer,\n",
        "    max_len_source,\n",
        "    max_len_target\n",
        "    )\n",
        "ds_valid = DateDataset(\n",
        "    x_val, \n",
        "    y_val,\n",
        "    tokenizer,\n",
        "    max_len_source,\n",
        "    max_len_target\n",
        "    )\n",
        "\n",
        "print(f'len ds_train: {len(ds_train)}')\n",
        "print(f'len ds_valid: {len(ds_valid)}')\n",
        "print(f'len ds_debug: {len(ds_debug)}')\n",
        "\n",
        "# dataloaders\n",
        "dataloaders = {\n",
        "    'debug': DataLoader(\n",
        "         ds_debug,\n",
        "         batch_size=BATCH_SZ,\n",
        "         shuffle=True,\n",
        "         num_workers=2,\n",
        "         pin_memory=True),\n",
        "    \n",
        "    'train': DataLoader(\n",
        "         ds_train,\n",
        "         batch_size=BATCH_SZ,\n",
        "         shuffle=True,\n",
        "         num_workers=2,\n",
        "         pin_memory=True),\n",
        "\n",
        "    'valid': DataLoader(\n",
        "         ds_valid,\n",
        "         batch_size=BATCH_SZ,\n",
        "         shuffle=False,\n",
        "         num_workers=2,\n",
        "         pin_memory=True)\n",
        "               }\n",
        "\n",
        "# sanity check\n",
        "dl_sizes = {x: len(dataloaders[x]) for x in dataloaders.keys()}; dl_sizes"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "len ds_train: 92809\n",
            "len ds_valid: 39776\n",
            "len ds_debug: 128\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'debug': 1, 'train': 726, 'valid': 311}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x3rqg6r7am-N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# testando o dataloader \n",
        "source_token_ids, source_mask, target_token_ids = next(iter(dataloaders['debug']))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wb4AnOJzBXK7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Net(torch.nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Net, self).__init__()\n",
        "        self.model = T5ForConditionalGeneration.from_pretrained(model_size)\n",
        "    \n",
        "    def forward(self, token_ids, att_mask, labels):\n",
        "        outputs = self.model.forward(\n",
        "            input_ids=token_ids, \n",
        "            attention_mask=att_mask,\n",
        "            lm_labels=labels\n",
        "            )\n",
        "        return outputs[0] # loss\n",
        "    \n",
        "    @torch.no_grad()    \n",
        "    def generate(self, token_ids, att_mask, max_len_target):\n",
        "        predict = self.model.generate(\n",
        "            input_ids=token_ids, \n",
        "            attention_mask=att_mask,\n",
        "            max_length=max_len_target\n",
        "            )\n",
        "        return predict"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tq9AMCaNuWMU",
        "colab_type": "text"
      },
      "source": [
        "## Funções de treino e eval"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v21m8GMfIUMb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5X6hklAUXxGD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def acc_in_text(trues, preds):\n",
        "    acc = []\n",
        "    for d in zip(trues, preds):\n",
        "        if d[0] == d[1]:\n",
        "            acc.append(1)\n",
        "        else:\n",
        "            acc.append(0)\n",
        "    return acc # bool\n",
        "\n",
        "def train(model, device, train_loader, optimizer):\n",
        "    loss_train = []\n",
        "    model.train()\n",
        "    for source_token_ids, source_mask, target_token_ids in train_loader:\n",
        "        optimizer.zero_grad()\n",
        "        loss = model(\n",
        "            source_token_ids.to(device), \n",
        "            source_mask.to(device), \n",
        "            target_token_ids.to(device)\n",
        "            )\n",
        "        \n",
        "        loss_train.append(loss.item())\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "  \n",
        "    return sum(loss_train) / len(loss_train)\n",
        "\n",
        "def evaluate_fn(model, device, val_loader, max_len=max_len_target):\n",
        "    all_acc, all_preds, all_trues = [], [], []\n",
        "    model.eval()\n",
        "    for source_token_ids, source_mask, target_token_ids in val_loader:\n",
        "        predicted_ids = model.generate(\n",
        "            source_token_ids.to(device), \n",
        "            source_mask.to(device),\n",
        "            max_len\n",
        "            )\n",
        "        \n",
        "        preds = [tokenizer.decode(t) for t in predicted_ids]\n",
        "        trues = [tokenizer.decode(t) for t in target_token_ids]\n",
        "        acc = acc_in_text(trues, preds)\n",
        "        all_acc.extend(acc)\n",
        "        all_trues.extend(trues)\n",
        "        all_preds.extend(preds)\n",
        "        \n",
        "    return np.array(all_acc).mean(), all_trues, all_preds"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pirS1mecELqp",
        "colab_type": "text"
      },
      "source": [
        "# Overfit em 1 batch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R2PxRYyfn_UO",
        "colab_type": "code",
        "outputId": "40ca8a33-eb6a-45b9-e2f2-fcd8ea980166",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "overfit = False\n",
        "\n",
        "if overfit:\n",
        "\n",
        "    start = torch.cuda.Event(enable_timing=True)\n",
        "    end = torch.cuda.Event(enable_timing=True)\n",
        "    deterministic() \n",
        "\n",
        "    model = Net().to(device)\n",
        "    optimizer = torch.optim.AdamW(model.parameters(), lr=5e-5)\n",
        "\n",
        "    N_EPOCHS  = 1000\n",
        "    WINDOW    = 7\n",
        "\n",
        "    # -----------------------------------------------------------------------------\n",
        "    start.record()\n",
        "    for step in range(1, N_EPOCHS+1):\n",
        "        samp = random.randint(0, BATCH_SZ-WINDOW) # to show random trues and preds\n",
        "        loss_t = train(model, device, dataloaders['debug'], optimizer)\n",
        "        acc, trues, preds = evaluate_fn(model, device, dataloaders['debug'])\n",
        "        if step == 1:\n",
        "            print(f'[Epoch: {step}/{N_EPOCHS}] |', end=' ')\n",
        "            print(f'Train Loss: {loss_t:.3f} -- Acc: {acc:.3f}')\n",
        "        if step % 50 == 0:\n",
        "            print(f'[Epoch: {step}/{N_EPOCHS}] |', end=' ')\n",
        "            print(f'Train Loss: {loss_t:.3f} -- Acc: {acc:.3f}')\n",
        "            print(f'  Trues: {trues[samp:samp+WINDOW]}\\n  Preds: {preds[samp:samp+WINDOW]}')\n",
        "    end.record()\n",
        "    torch.cuda.synchronize()    \n",
        "    # -----------------------------------------------------------------------------\n",
        "\n",
        "    print(f'Tempo: {start.elapsed_time(end)/1000/60 :.3f} min.')\n",
        "    del model"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Deterministic experiment, seed: 0\n",
            "[Epoch: 1/1000] | Train Loss: 12.042 -- Acc: 0.000\n",
            "[Epoch: 50/1000] | Train Loss: 1.969 -- Acc: 0.000\n",
            "  Trues: ['23/11/1969', '13/12/1969', '24/11/1950', '09/03/2008', '05/05/2013', '26/04/1984', '05/12/2011']\n",
            "  Preds: ['', '', '', '', '', '', '']\n",
            "[Epoch: 100/1000] | Train Loss: 1.195 -- Acc: 0.133\n",
            "  Trues: ['09/06/1984', '07/07/1910', '23/05/1958', '20/06/1986', '19/06/1926', '07/01/1917', '27/12/1937']\n",
            "  Preds: [' ⁇   ⁇   ⁇   ⁇   ⁇  ', ' ⁇   ⁇   ⁇   ⁇   ⁇  ', ' ⁇   ⁇   ⁇   ⁇   ⁇  ', ' ⁇   ⁇   ⁇   ⁇   ⁇  ', '', ' ⁇   ⁇   ⁇   ⁇   ⁇  ', '']\n",
            "[Epoch: 150/1000] | Train Loss: 0.873 -- Acc: 0.180\n",
            "  Trues: ['21/10/1954', '13/04/1951', '26/04/1984', '20/08/1978', '07/07/1910', '23/11/1969', '05/12/2011']\n",
            "  Preds: ['21/10/1954', '15/06/1988', '05/07/1983', '05/08/1999', '05/07/1999', '05/07/1988', '05/08/1920']\n",
            "[Epoch: 200/1000] | Train Loss: 0.783 -- Acc: 0.195\n",
            "  Trues: ['05/02/1918', '12/06/1976', '12/03/2020', '22/07/1996', '07/07/1910', '09/06/1984', '05/08/1982']\n",
            "  Preds: ['05/06/1999', '12/10/1946', '12/03/1920', '27/07/1988', '12/07/1922', '11/07/1942', '05/07/1988']\n",
            "[Epoch: 250/1000] | Train Loss: 0.670 -- Acc: 0.266\n",
            "  Trues: ['22/07/1996', '02/11/1988', '31/07/1971', '06/10/2003', '01/04/1991', '21/12/1940', '21/10/1954']\n",
            "  Preds: ['27/07/1946', '11/08/1968', '31/07/1988', '31/08/1983', '01/04/1991', '21/12/1940', '21/10/1954']\n",
            "[Epoch: 300/1000] | Train Loss: 0.586 -- Acc: 0.320\n",
            "  Trues: ['20/08/1978', '27/12/1937', '27/09/1926', '14/11/1910', '21/12/1940', '23/05/1991', '28/05/1961']\n",
            "  Preds: ['27/08/1988', '27/12/1937', '27/04/1916', '14/11/1920', '21/12/1940', '23/07/1941', '28/05/1961']\n",
            "[Epoch: 350/1000] | Train Loss: 0.495 -- Acc: 0.391\n",
            "  Trues: ['18/01/1915', '21/07/1914', '09/09/2019', '14/11/1910', '09/07/2013', '18/01/1908', '20/08/1978']\n",
            "  Preds: ['22/08/1915', '27/07/1929', '06/07/1964', '14/11/1910', '09/07/2013', '10/10/1978', '25/08/1978']\n",
            "[Epoch: 400/1000] | Train Loss: 0.423 -- Acc: 0.461\n",
            "  Trues: ['10/01/1994', '05/03/1998', '21/11/2011', '24/04/1948', '13/08/1937', '06/01/2009', '20/12/1958']\n",
            "  Preds: ['10/10/1934', '05/03/1998', '23/11/2011', '24/04/1948', '02/08/1937', '06/01/2009', '21/12/1958']\n",
            "[Epoch: 450/1000] | Train Loss: 0.349 -- Acc: 0.633\n",
            "  Trues: ['15/03/1951', '23/05/1958', '15/12/1932', '10/11/1928', '08/05/1973', '19/06/1926', '20/12/1958']\n",
            "  Preds: ['15/03/1951', '23/05/1958', '15/12/1932', '10/11/1928', '08/05/1973', '17/06/1926', '20/12/1958']\n",
            "[Epoch: 500/1000] | Train Loss: 0.269 -- Acc: 0.750\n",
            "  Trues: ['28/05/1961', '01/04/1991', '13/08/1937', '18/01/1915', '05/10/1966', '14/03/2003', '05/05/2013']\n",
            "  Preds: ['28/05/1961', '01/04/1991', '13/08/1937', '19/01/1915', '05/10/1966', '14/03/2003', '05/05/2013']\n",
            "[Epoch: 550/1000] | Train Loss: 0.205 -- Acc: 0.859\n",
            "  Trues: ['10/10/1976', '22/07/1996', '23/12/1991', '28/11/1916', '27/12/1937', '09/06/1984', '14/11/2010']\n",
            "  Preds: ['10/10/1976', '22/07/1996', '23/12/1991', '28/11/1916', '27/12/1937', '09/06/1984', '14/11/2010']\n",
            "[Epoch: 600/1000] | Train Loss: 0.175 -- Acc: 0.922\n",
            "  Trues: ['27/09/1926', '12/10/2004', '13/04/1951', '29/08/1932', '02/10/1927', '25/07/1975', '24/11/1950']\n",
            "  Preds: ['27/09/1926', '12/10/2004', '13/04/1951', '29/08/1932', '02/10/1927', '25/07/1975', '24/11/1950']\n",
            "[Epoch: 650/1000] | Train Loss: 0.132 -- Acc: 0.977\n",
            "  Trues: ['24/11/2003', '13/04/1951', '27/09/1926', '28/04/1967', '24/05/2003', '11/06/1915', '14/11/1934']\n",
            "  Preds: ['24/11/2003', '13/04/1951', '27/09/1926', '28/04/1967', '24/05/2003', '11/06/1915', '14/11/1934']\n",
            "[Epoch: 700/1000] | Train Loss: 0.115 -- Acc: 0.977\n",
            "  Trues: ['16/10/2011', '21/07/1914', '11/11/1954', '04/10/1940', '13/12/1969', '26/07/1981', '08/12/1903']\n",
            "  Preds: ['16/10/2011', '21/07/1914', '11/11/1954', '04/10/1940', '13/12/1969', '26/07/1981', '08/12/1903']\n",
            "[Epoch: 750/1000] | Train Loss: 0.082 -- Acc: 0.984\n",
            "  Trues: ['21/10/1954', '20/08/1978', '07/05/1948', '12/10/2004', '27/09/1926', '07/01/2011', '07/05/2007']\n",
            "  Preds: ['21/10/1954', '20/08/1978', '07/05/1948', '12/10/2004', '27/09/1926', '07/01/2011', '07/05/2007']\n",
            "[Epoch: 800/1000] | Train Loss: 0.066 -- Acc: 0.992\n",
            "  Trues: ['12/10/2004', '20/03/1979', '15/12/1930', '02/05/1986', '25/07/1975', '07/01/2011', '03/07/1966']\n",
            "  Preds: ['12/10/2004', '20/03/1979', '15/12/1930', '02/05/1986', '25/07/1975', '07/01/2011', '03/07/1966']\n",
            "[Epoch: 850/1000] | Train Loss: 0.063 -- Acc: 1.000\n",
            "  Trues: ['19/06/1926', '15/11/1976', '04/05/1973', '15/12/1930', '16/10/2011', '14/11/1910', '30/10/1964']\n",
            "  Preds: ['19/06/1926', '15/11/1976', '04/05/1973', '15/12/1930', '16/10/2011', '14/11/1910', '30/10/1964']\n",
            "[Epoch: 900/1000] | Train Loss: 0.053 -- Acc: 1.000\n",
            "  Trues: ['14/03/2003', '09/07/2013', '23/05/1991', '28/04/1967', '23/05/2008', '15/11/1949', '21/12/1940']\n",
            "  Preds: ['14/03/2003', '09/07/2013', '23/05/1991', '28/04/1967', '23/05/2008', '15/11/1949', '21/12/1940']\n",
            "[Epoch: 950/1000] | Train Loss: 0.039 -- Acc: 1.000\n",
            "  Trues: ['11/10/1941', '05/02/1918', '31/08/1941', '28/09/1980', '15/12/1930', '24/10/1968', '28/05/1961']\n",
            "  Preds: ['11/10/1941', '05/02/1918', '31/08/1941', '28/09/1980', '15/12/1930', '24/10/1968', '28/05/1961']\n",
            "[Epoch: 1000/1000] | Train Loss: 0.031 -- Acc: 1.000\n",
            "  Trues: ['21/11/2011', '20/12/1958', '04/10/1940', '31/08/1928', '23/12/1991', '14/10/1900', '17/07/1978']\n",
            "  Preds: ['21/11/2011', '20/12/1958', '04/10/1940', '31/08/1928', '23/12/1991', '14/10/1900', '17/07/1978']\n",
            "Tempo: 10.947 min.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wQC8YJ8PT7-0",
        "colab_type": "text"
      },
      "source": [
        "# Treino completo"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dNtIb8b4Q4lC",
        "colab_type": "code",
        "outputId": "d8e6ebaa-930c-407c-cc4d-b87332fc306c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        }
      },
      "source": [
        "start = torch.cuda.Event(enable_timing=True)\n",
        "end = torch.cuda.Event(enable_timing=True)\n",
        "deterministic() \n",
        "\n",
        "model = Net().to(device)\n",
        "optimizer = torch.optim.AdamW(model.parameters(), lr=5e-5)\n",
        "\n",
        "N_EPOCHS  = 4\n",
        "WINDOW    = 7\n",
        "\n",
        "# ---------------------------------------------------------------------------------\n",
        "start.record()\n",
        "\n",
        "for step in range(1, N_EPOCHS+1):\n",
        "    samp = random.randint(0, BATCH_SZ-WINDOW) # to show random trues and preds\n",
        "    loss_t = train(model, device, dataloaders['train'], optimizer)\n",
        "    acc, trues, preds = evaluate_fn(model, device, dataloaders['valid'])\n",
        "    print(f'[Epoch: {step}/{N_EPOCHS}] |', end=' ')\n",
        "    print(f'Train Loss: {loss_t:.3f} -- Acc: {acc:.3f}')\n",
        "    print(f'  Trues: {trues[samp:samp+WINDOW]}\\n  Preds: {preds[samp:samp+WINDOW]}')\n",
        "\n",
        "end.record()\n",
        "torch.cuda.synchronize()    \n",
        "# ---------------------------------------------------------------------------------\n",
        "\n",
        "print(f'Tempo: {start.elapsed_time(end)/1000/60 :.3f} min.')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Deterministic experiment, seed: 0\n",
            "[Epoch: 1/4] | Train Loss: 0.852 -- Acc: 0.819\n",
            "  Trues: ['20/04/1943', '22/02/1927', '01/09/1972', '02/01/1905', '17/04/2008', '28/03/1928', '17/07/1961']\n",
            "  Preds: ['20/04/1943', '22/02/1927', '01/09/1972', '02/01/1905', '17/04/08', '28/03/1928', '17/07/1962']\n",
            "[Epoch: 2/4] | Train Loss: 0.082 -- Acc: 0.998\n",
            "  Trues: ['16/03/2006', '12/01/2019', '23/10/1972', '25/10/2004', '08/01/1986', '08/05/1953', '27/05/1961']\n",
            "  Preds: ['16/03/2006', '12/01/2019', '23/10/1972', '25/10/2004', '08/01/1986', '08/05/1953', '27/05/1961']\n",
            "[Epoch: 3/4] | Train Loss: 0.023 -- Acc: 0.999\n",
            "  Trues: ['27/05/1961', '09/09/1969', '01/01/1975', '07/04/1912', '01/06/1901', '17/07/1992', '10/11/2010']\n",
            "  Preds: ['27/05/1961', '09/09/1969', '01/01/1975', '07/04/1912', '01/06/1901', '17/07/1992', '10/11/2010']\n",
            "[Epoch: 4/4] | Train Loss: 0.011 -- Acc: 1.000\n",
            "  Trues: ['23/01/1994', '01/11/1914', '03/12/1977', '11/08/1968', '02/06/1935', '25/02/1934', '09/08/1915']\n",
            "  Preds: ['23/01/1994', '01/11/1914', '03/12/1977', '11/08/1968', '02/06/1935', '25/02/1934', '09/08/1915']\n",
            "Tempo: 13.034 min.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CzVjYAkNGVxJ",
        "colab_type": "text"
      },
      "source": [
        "# Teste de predição"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yQPXEekhM4xS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "example_0  = 'um de jan. de mil, dois mil e dez'\n",
        "example_1 = '01.01.2050'\n",
        "\n",
        "s = tokenizer.encode_plus(\n",
        "    f'{example_0} {tokenizer.eos_token}',\n",
        "    max_length=max_len_source,\n",
        "    pad_to_max_length=True,\n",
        "    return_tensors='pt')\n",
        "s.keys()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cb2oUfVV0BIc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.eval()\n",
        "predicted_tokens = model.generate(\n",
        "    s.input_ids.to(device), \n",
        "    s.attention_mask.to(device),\n",
        "    )\n",
        "predict = [tokenizer.decode(t) for t in predicted_tokens]; predict[0]"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}