Copy_of_BERT_with_TF.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of BERT with TF.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "472b83dc33dd40b6b959be59d8b76820": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0e817c63c9184e87aa69dc8bd18eb0f2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_866b304566414cb0aa941474fb04c9b4",
              "IPY_MODEL_873b989977db41a6b5cada5b27685590"
            ]
          }
        },
        "0e817c63c9184e87aa69dc8bd18eb0f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "866b304566414cb0aa941474fb04c9b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_031af9f5d26442f89b01582266336db1",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e07bddefcf7144d5a82830bc4dbde632"
          }
        },
        "873b989977db41a6b5cada5b27685590": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fe3eaa1720d34bd9b72352fc2ebd03f7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:00&lt;00:00, 8.70kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2e19deb6c9a24fccbfe511039ba56c12"
          }
        },
        "031af9f5d26442f89b01582266336db1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e07bddefcf7144d5a82830bc4dbde632": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fe3eaa1720d34bd9b72352fc2ebd03f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2e19deb6c9a24fccbfe511039ba56c12": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6bfcd01677c149759b7c8d98e58a13ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8eeec842278449e1bf68ece647c13b0b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e49adf371c634d0bbedc2c6b95666f78",
              "IPY_MODEL_e830d6646d0c4956939f0c3b512acdb2"
            ]
          }
        },
        "8eeec842278449e1bf68ece647c13b0b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e49adf371c634d0bbedc2c6b95666f78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4fdee1e3bfea4c27866df41bad148934",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 536063208,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 536063208,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a3915728defd4c0282964e55433f2845"
          }
        },
        "e830d6646d0c4956939f0c3b512acdb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_493a64c8a8c24906bd4a220127008206",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 536M/536M [00:14&lt;00:00, 38.2MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_41ccad1e183144d39c992c6a8c3bf935"
          }
        },
        "4fdee1e3bfea4c27866df41bad148934": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a3915728defd4c0282964e55433f2845": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "493a64c8a8c24906bd4a220127008206": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "41ccad1e183144d39c992c6a8c3bf935": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3e415d9caffa4a76b75556032aaa640a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d156613fb3d1437fb54a03b346396929",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_508c6378c6af4f4885a9ad5543378a42",
              "IPY_MODEL_52c876a7a123480a81acd319489efec5"
            ]
          }
        },
        "d156613fb3d1437fb54a03b346396929": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "508c6378c6af4f4885a9ad5543378a42": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a7288fae6af348419a2f75efb64080eb",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_19cffaead63c47d4b2eb7f02ce3a6a50"
          }
        },
        "52c876a7a123480a81acd319489efec5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e5fae09ea1a34e5aba94f84a6f62a734",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 634kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_71ed185d60854ba7a2b411c443e47b4e"
          }
        },
        "a7288fae6af348419a2f75efb64080eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "19cffaead63c47d4b2eb7f02ce3a6a50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e5fae09ea1a34e5aba94f84a6f62a734": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "71ed185d60854ba7a2b411c443e47b4e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/MOTURUPRAVEENBHARGAV/-Classification-models-on-purchase_prediction-Data/blob/master/Copy_of_BERT_with_TF.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TpE8FL5O6-bp"
      },
      "source": [
        "### Installing Transformers\r\n",
        "Installing the Transformers library is fairly easy. Just run the following pip line on a Google Colab cell:\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1n1Nm4Qw5WiD",
        "outputId": "420e4db3-7668-42b4-f480-163838777c9a"
      },
      "source": [
        "pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f9/54/5ca07ec9569d2f232f3166de5457b63943882f7950ddfcc887732fc7fb23/transformers-4.3.3-py3-none-any.whl (1.9MB)\n",
            "\u001b[K     |████████████████████████████████| 1.9MB 5.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/23/2ddc317b2121117bf34dd00f5b0de194158f2a44ee2bf5e47c7166878a97/tokenizers-0.10.1-cp37-cp37m-manylinux2010_x86_64.whl (3.2MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 5.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 7.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp37-none-any.whl size=893262 sha256=950f51d642751be18338fc057117cd1d71083cf2daabd0ca09785ff90a5d9640\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.10.1 transformers-4.3.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wiVRGlRe7MYH"
      },
      "source": [
        "After the installation is completed, we will load the pre-trained BERT Tokenizer and Sequence Classifier as well as InputExample and InputFeatures. Then, we will build our model with the Sequence Classifier and our tokenizer with BERT’s Tokenizer."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 232,
          "referenced_widgets": [
            "472b83dc33dd40b6b959be59d8b76820",
            "0e817c63c9184e87aa69dc8bd18eb0f2",
            "866b304566414cb0aa941474fb04c9b4",
            "873b989977db41a6b5cada5b27685590",
            "031af9f5d26442f89b01582266336db1",
            "e07bddefcf7144d5a82830bc4dbde632",
            "fe3eaa1720d34bd9b72352fc2ebd03f7",
            "2e19deb6c9a24fccbfe511039ba56c12",
            "6bfcd01677c149759b7c8d98e58a13ed",
            "8eeec842278449e1bf68ece647c13b0b",
            "e49adf371c634d0bbedc2c6b95666f78",
            "e830d6646d0c4956939f0c3b512acdb2",
            "4fdee1e3bfea4c27866df41bad148934",
            "a3915728defd4c0282964e55433f2845",
            "493a64c8a8c24906bd4a220127008206",
            "41ccad1e183144d39c992c6a8c3bf935",
            "3e415d9caffa4a76b75556032aaa640a",
            "d156613fb3d1437fb54a03b346396929",
            "508c6378c6af4f4885a9ad5543378a42",
            "52c876a7a123480a81acd319489efec5",
            "a7288fae6af348419a2f75efb64080eb",
            "19cffaead63c47d4b2eb7f02ce3a6a50",
            "e5fae09ea1a34e5aba94f84a6f62a734",
            "71ed185d60854ba7a2b411c443e47b4e"
          ]
        },
        "id": "1oAhl0KQ6zfU",
        "outputId": "76bcf0ba-0458-4b8d-d1f4-7057cc7f10d1"
      },
      "source": [
        "from transformers import BertTokenizer, TFBertForSequenceClassification\r\n",
        "from transformers import InputExample, InputFeatures\r\n",
        "\r\n",
        "model = TFBertForSequenceClassification.from_pretrained(\"bert-base-uncased\")\r\n",
        "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "472b83dc33dd40b6b959be59d8b76820",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6bfcd01677c149759b7c8d98e58a13ed",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=536063208.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "All model checkpoint layers were used when initializing TFBertForSequenceClassification.\n",
            "\n",
            "Some layers of TFBertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3e415d9caffa4a76b75556032aaa640a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LZM9ADCX8l7r"
      },
      "source": [
        "Let’s see the summary of our BERT model:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "91cdSXsz9GbB"
      },
      "source": [
        "Here are the results. We have the main BERT model, a dropout layer to prevent overfitting, and finally a dense layer for classification task:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-UE1a7T_8bOg",
        "outputId": "5df3b42e-eabb-4d5c-8de2-a826ad3edf88"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"tf_bert_for_sequence_classification\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "bert (TFBertMainLayer)       multiple                  109482240 \n",
            "_________________________________________________________________\n",
            "dropout_37 (Dropout)         multiple                  0         \n",
            "_________________________________________________________________\n",
            "classifier (Dense)           multiple                  1538      \n",
            "=================================================================\n",
            "Total params: 109,483,778\n",
            "Trainable params: 109,483,778\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 77
        },
        "id": "YLHhdwPh8vmM",
        "outputId": "e47dfa16-f38c-4d22-ad46-81d086c33aa4"
      },
      "source": [
        "import tensorflow as tf\r\n",
        "tf.keras.utils.plot_model(model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAA8CAIAAAC8fHszAAAABmJLR0QA/wD/AP+gvaeTAAAKOUlEQVR4nO3dfUhT3xsA8LPc3N00ncu3NFupkdGk1CQNg4ZCQlBoSoaB9Y+VUEJZglpIUKJlCmYvQgQZ+TUtBCskekNBXYaGla0sclZWmmvzZctt7nz/uDTudy/XbUxPv1/P5z/PPXvOc849j7vbdZODMUYAAKIWkU4AAAB1CMAfAOoQAPKgDgEgj8v8oaur6/z586RSAeDvkZiYeOTIEfOP/3k+/PTpU3Nz84KnBMDfpbu7u6uri9nCte7U1NS0UPkA8DfKzMy0aIHXhwCQB3UIAHlQhwCQB3UIAHlQhwCQB3UIAHlQhwCQB3UIAHlQhwCQB3UIAHlQhwCQB3UIAHlQhwCQB3UIAHnuqcOZmZn8/Pzg4GChUNjW1sbSMz4+3sPDY/369W4Z1ymOJwlccO7cucDAQA6Hc/nyZXfFvH//vq+vb2trq7nF4iRad3DvcAvGPXVYWVnZ1tamUCiqq6unpqZYevb09MhkMrcM6izHkwQuKCgo6OzsdG9M6y/1tDiJ7v3WT4LfIWrjc8Bz0ul0ycnJzEVvaWnZsGGDSCTKzc11JAKHw3Fh3DnTYOdskoC4bdu2aTQaZov1SbTo4BSLLWQ93IJx5fnw6tWro6OjzJbPnz/zeDzHIzjV2fE02Dmb5EJSKpU6nY50Fv8D3HsSnd1C8wgzNDY2WrRYy8/P9/T0pB8bERHx4MGDiIgIczQvLy/2h2OMk5OT/fz8Vq9eLRQKKYpKSkrq6OgwHzUajSdOnAgLC6MoKjo6+p9//sEYl5eXCwQCb2/v79+/HzlyJCQkJDU1lZkG+4g2kzSZTJWVlVFRUZ6eniKRaMeOHW/evKH7Ww+nUCjYh3j69Gl8fLxAIFi8eLFUKtVoNPbmQg9dUVGxatUqHo/n6+u7Zs0aDofT19eHMT506BCPxwsKCqJ75uXlCYVChNDY2BjL+tTW1gqFQoFA0NLSkpqaunjx4tDQ0Js3bzIzvH79elxcHJ/PFwqFEonk1KlTLBnOyWa0wcFBhNClS5foPu3t7WvWrPHx8eHz+VKptK2tjWWtrBs7OjrCwsIQQjU1NTZPokUHlsRsZmKxk62jsewQRxacRUZGRkZGBrPF6TrEGO/cudNi6wcFBeXk5DiYRHJycnh4+MePHw0Gw6tXrzZu3EhR1Lt37+ijBQUFfD6/ubn558+fRUVFixYt6unpwRgXFxcjhPLz82tqatLT09+8eWOdBjuLJE+ePOnp6VlfX69Wq/v7+2NjY/39/b99+0YftR6OJfLU1JSPj095eblOp/v27Vt6ejpdNvbmcubMGQ6HU1FRoVKptFrthQsXEEJ0HWKMs7OzzXWIMT579iyzDtnX59GjRxqNZnR0dPPmzV5eXnq9nn5UVVUVQqisrGx8fFylUl25ciU7O5slGjt70SzqsKmpqbS0VKVSjY+PJyQkLFmyxN5a2VvAT58+WZSZxUm06GAvMZuZYKudbBHNkR1ib8HZ/Sl1uG7dOvOP/f39CKGCggKMsU6nEwqFWVlZ9CGtVsvn8/Py8vDvaet0OpY02DGT1Gq13t7e5oEwxs+ePUMI0b8+bQ7H4tWrVwihu3fvMhvtzWV6elokEqWkpJh7NjQ0OFiHjq9PbW0tQuj9+/cYY71eLxKJZDKZOabRaKyurmaJxsJeNGxVh0xnzpxBCI2OjtpcK5uN2Mk6ZEnMZiaYtQ6d3SHMBZ+TdR2Sv38YHR3t6+tLV+Pbt2+1Wq1UKqUPCQSC4OBghULh9kFfv349NTW1YcMGc0t8fLynp6dcLnchWnh4eGBg4J49e0pLS4eGhuhGe3MZHBxUq9UpKSkuDOT4+tBXXAaDASHU39+vVqu3bt1qPurh4ZGfn+/aatuLxv4o+kXd7OyszbWy2egsBxMzZ8IezdkdwlxwF5CvQ4QQj8ejJzA9PY0QKikp4fymVCq1Wq3bR1Sr1Qghb29vZqNIJJqcnHQhmkAgePz4cVJS0unTp8PDw7OysnQ6nb25fP36FSEUEBDgwkCurc/ExARCSCQSzWs0a/fu3duyZUtAQACfzz9+/DjdaHOtbDbOGd/xxGxmws69O2RO5OvQaDSqVKrly5ej37uzqqqK+ZRt8Y2rbkGfLYs1VavVy5Ytcy3g2rVrW1tbR0ZGCgsLGxsbz507Z28u/v7+6PdpdpZr6xMSEoIQ+vHjx7xGszA8PJyWlhYcHCyXyzUaTXl5ufmQ9VrZa3SKvcRYMmHh9h3CjnwdPnnyxGQyxcbGIoToN+5evHgx34NKpVJvb+/nz5+bW+RyuV6vj4uLcyHayMjIwMAAQiggIKCsrCw2NnZgYMDeXCIjI/l8fnd3t71oXC7X3uWNa+uzYsUKsVj84MGDeY1m4eXLlwaDIS8vLzw8nKIo8x1jm2tls9GprFgSs5cJO/fukDm5UodisXhkZGRoaGhyctK1C2K9Xq/RaIxGY29v7+HDhyUSyd69exFCFEXt27evoaHh4sWLExMTs7Oznz9/pi/k3JsGRVFHjx69c+fOjRs3JiYmXr58efDgwaVLl+7fv9+F6YyMjBw4cEChUOj1+r6+PqVSmZCQYG8uIpEoJyfnzp07dXV1k5OTWq1WqVQyo0VGRqpUqpaWFoPBMDY2xjzq1PqY8fn8oqKi9vb2w4cPf/nyxWQyTU5ODgwMuDeaRTf6Aufhw4e/fv0aHBw0v6yyuVY2Gx1c/DkTs5cJYt1C7t0hc2Nekzj4fmlvb69EIhEIBElJSXK5PCYmBiHE5XJjY2Obm5vnfPi1a9dkMllgYCCXy12yZMnu3buVSqX56MzMTGFh4fLly7lcbkBAwM6dO1+/fk3f0EMIhYWF1dfXW6dhfjfZpqGhIeskTSbT2bNn6Zt4fn5+aWlpb9++pfvbHI49/qZNm/z8/Dw8PEJCQoqLi41Go725YIynpqZyc3P9/f25XK5YLI6KikKM90vHx8dlMhlFUStXrjx06NCxY8cQQpGRkcPDw/Zi0rezEEKrVq368OFDXV2dj48PQkgikZhvCF24cCE6OpqiKIqiYmJiamtrWTKck3W0ysrKoKAghJCXl1d6ejrGuLCwUCwWi0SizMxM+t4MfZvOeq1sLmBNTU1wcDBCSCgUbt++3fokWnRgmabNTIaHh5lbqKSkxCIayw5xZMFZWL9fysGMv6m7devWrl27MPyn7oV1+/btjIyMvr4+In/+DhYe/f8tmP9IhvzrQ+Dym93g/4ab61ChUHDsy8rKcu9wCzMokUmR8ldN9s/hyuctWERFRS38Ze18Dzqv8evq6ug7Wjt27Ojs7AwNDZ2ngRxE5AwCuC4lLDc3V61WY4yVSiXxIgSkQB0CQB7UIQDkQR0CQB7UIQDkQR0CQB7UIQDkQR0CQB7UIQDkQR0CQB7UIQDkQR0CQB7UIQDkQR0CQJ6Nzz3RHxYGAMyT7u5uiy/g+c/zYVhYWEZGxsKmBMBfJyEhITExkdnCgQ99AkAcvD4EgDyoQwDIgzoEgDyoQwDI+xc8avaQ/hR2AAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<IPython.core.display.Image object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BV4NDNuu9LOe"
      },
      "source": [
        "Now that we have our model, let’s create our input sequences from the IMDB reviews dataset:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B_YNcQHA9uct"
      },
      "source": [
        "IMDB Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o5SRYrUj90_B"
      },
      "source": [
        "IMDB Reviews Dataset is a large movie review dataset collected and prepared by Andrew L. Maas from the popular movie rating service, IMDB. The IMDB Reviews dataset is used for binary sentiment classification, whether a review is positive or negative. It contains 25,000 movie reviews for training and 25,000 for testing. All these 50,000 reviews are labeled data that may be used for supervised deep learning. Besides, there is an additional 50,000 unlabeled reviews that we will not use in this case study. In this case study, we will only use the training dataset."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QjSPR5Qt97rz"
      },
      "source": [
        "Initial Imports"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0ne8VC0A-Bp7"
      },
      "source": [
        "We will first have two imports: TensorFlow and Pandas."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rtHLkI2O9yMz"
      },
      "source": [
        "import tensorflow as tf\r\n",
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1xiOVUwG-N6G"
      },
      "source": [
        "Get the Data from the Stanford Repo"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LWnqfOjl-Was"
      },
      "source": [
        "Then, we can download the dataset from Stanford’s relevant directory with *tf.keras.utils.get_file* function, as shown below:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VyXxS2Pd-KoC",
        "outputId": "14047f82-fdec-473a-9df5-66760b3958a4"
      },
      "source": [
        "URL = \"https://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\"\r\n",
        "\r\n",
        "dataset = tf.keras.utils.get_file(fname=\"aclImdb_v1.tar.gz\", \r\n",
        "                                  origin=URL,\r\n",
        "                                  untar=True,\r\n",
        "                                  cache_dir='.',\r\n",
        "                                  cache_subdir='')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\n",
            "84131840/84125825 [==============================] - 9s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4NgyKtWI-qAH",
        "outputId": "55a0c9ee-2641-4c5a-ca0e-fb7b60072f71"
      },
      "source": [
        "\r\n",
        "# The shutil module offers a number of high-level \r\n",
        "# operations on files and collections of files.\r\n",
        "import os\r\n",
        "import shutil\r\n",
        "# Create main directory path (\"/aclImdb\")\r\n",
        "main_dir = os.path.join(os.path.dirname(dataset), 'aclImdb')\r\n",
        "# Create sub directory path (\"/aclImdb/train\")\r\n",
        "train_dir = os.path.join(main_dir, 'train')\r\n",
        "# Remove unsup folder since this is a supervised learning task\r\n",
        "remove_dir = os.path.join(train_dir, 'unsup')\r\n",
        "shutil.rmtree(remove_dir)\r\n",
        "# View the final train folder\r\n",
        "print(os.listdir(train_dir))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['neg', 'urls_unsup.txt', 'urls_pos.txt', 'pos', 'labeledBow.feat', 'urls_neg.txt', 'unsupBow.feat']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ti_Zk8He_f95"
      },
      "source": [
        "## **Train and Test Split**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_4zN8guJABZB"
      },
      "source": [
        "Now that we have our data cleaned and prepared, we can create *text_dataset_from_directory* with the following lines. I want to process the entire data in a single batch. That’s why I selected a very large batch size:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "caE0EZBM_cWU",
        "outputId": "43622e1b-b123-40e1-d308-4ac099ab4b59"
      },
      "source": [
        "\r\n",
        "# We create a training dataset and a validation \r\n",
        "# dataset from our \"aclImdb/train\" directory with a 80/20 split.\r\n",
        "train = tf.keras.preprocessing.text_dataset_from_directory(\r\n",
        "    'aclImdb/train', batch_size=30000, validation_split=0.2, \r\n",
        "    subset='training', seed=123)\r\n",
        "test = tf.keras.preprocessing.text_dataset_from_directory(\r\n",
        "    'aclImdb/train', batch_size=30000, validation_split=0.2, \r\n",
        "    subset='validation', seed=123)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 25000 files belonging to 2 classes.\n",
            "Using 20000 files for training.\n",
            "Found 25000 files belonging to 2 classes.\n",
            "Using 5000 files for validation.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DWIzGG3vDSDp"
      },
      "source": [
        "## Convert to Pandas to View and Process"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dMHA9twnDSI4"
      },
      "source": [
        "Now we have our basic train and test datasets, I want to prepare them for our BERT model. To make it more comprehensible, I will create a pandas dataframe from our TensorFlow dataset object. The following code converts our train Dataset object to train pandas dataframe:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sSKi7_6HCO-y",
        "outputId": "7d3d605c-54ab-4d4e-dacf-4f732dfbcbf9"
      },
      "source": [
        "for i in train.take(1):\r\n",
        "  print(i)\r\n",
        "  train_feat = i[0].numpy()\r\n",
        "  train_lab = i[1].numpy()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(<tf.Tensor: shape=(20000,), dtype=string, numpy=\n",
            "array([b\"Canadian director Vincenzo Natali took the art-house circuit by storm with the intriguing and astonishingly intelligent Cube, which is my personal favourite SF film of the 90s. It framed the basic conceit of a group of strangers trapped in a maze shaped like a giant cube, shot entirely on one set, and took this idea in fascinating directions. <br /><br />I've been eagerly awaiting Natali's follow-up, and although its taken five years for him to mount another project, I'm delighted to say it was worth the wait. Cypher is a fascinating exploration of one man's place in the world, and how through a completely logical chain of events, finds himself in a situation beyond his control.<br /><br />I don't want to reveal too much about the plot, because one of the joys of Cypher is the different avenues it takes us down. It is so refreshing in this day and age to see a SF film that has more than one idea in it's head. Cypher is such a film.<br /><br />Morgan Sullivan (Jeremy Northam), one of the blandest people to ever walk the planet, is hired by the company DigiCorp. They send him to different parts of America to record different seminars. To his bewilderment, they are unbelievably boring. Covering topics as mundane as shaving cream and cheese.<br /><br />While Morgan is waiting for one seminar, he runs into Rita Foster (an impeccably cast Lucy Liu), the definition of an ice maiden. She gives him the brush-off, but there is something to her he finds irresistible. That's not too surprising considering the dry marriage he is in. <br /><br />When Rita turns up at another one of Morgan's seminars, she tells him his life is not what it appears. And I'm not saying anything more about the plot. To do so would cheapen the impact the rest of the film has on us, as well as the tortuous path that's so much fun to follow.<br /><br />As with Cube, Natali shows quite a talent for encompassing seemingly ordinary people, taking them out of the familiar, and basically seeing what will happen when they're thrust into the unknown. And Cypher follows similar patterns. But it's not a carbon copy of Cube. It has it's own inspiration.<br /><br />Cypher is a film that has more in common with conspiracy thrillers and paranoia stories. One of the great things about Cypher is the way these themes creep into the story without your knowledge. When Morgan realises his false identity is a piece of a much larger puzzle, it's as much of a shock to us as it is to him.<br /><br />One thing that distinguishes Cypher from Cube is how much more polished it is. Where Cube was confined to a minimalist setting and a shoestring budget with a cast of unknowns, Cypher is also on a low budget, but Natali economises it as much as he can, allowing him to broaden the horizon, and launching Morgan on an amazing journey through the labyrinth of his own identity.<br /><br />Natali's direction is exceptional, with a deft hand on the reins. There are some amazing camera angles from above, such as the enormity of the DigiCorp building as a vast, robust office block in conjunction to the insignificant speck that is Morgan standing outside. All the colour appears to have been bled out of the picture, which compliments the tone of the film perfectly as a modern day film-noir.<br /><br />The acting is uniformly excellent throughout. Jeremy Northam is a sympathetic figure from his loveless marriage to questioning his own identity. His performance is excellent because it's so modulated. He literally seems to transform right before our very eyes. From a clinical, spineless wimp to a confident man who will do anything to preserve his new identity.<br /><br />David Hewlett puts in a welcome appearance who made such an impact in Cube. He resides in a secret silo that looks like it was borrowed from Men in Black. His scene is one of the best because it's an exercise in carefully calculated suspense and paranoia. He is a supposed expert in identifying double-agents, and it's a fantastic piece of writing, brilliantly acted by Hewlett. All he has to do is look at Morgan, and we're drawn into his complex mind game.<br /><br />But it's Lucy Liu who's the scene stealer here. Too often she is cast in films where her potential is not utilised to full effect. But in Cypher, she is finally given a character that fits her like a glove. Rita is an aloof, guarded femme fatale that Liu inhabits with relish. I perked up every time she appeared because she is always in control, and can reduce a room to silence by the power of her icy stare alone.<br /><br />Things come to a very gratifying end, that doesn't conclude on an ambiguous note the way Cube did. But Morgan deserves his happy ending. After he's been put through the ringer like this, I cheered for him in the final scene. It's a perfect final moment because it comes as a ray of sunshine after a gloomy 90 minutes.<br /><br />Cypher succeeds on all counts. Engaging, shocking, always entertaining, it's everything that Total Recall wanted to be but wasn't. And it comes as a refreshing antidote to the overwhelming and inexplicable Matrix.<br /><br />A fine follow-up from Natali. And now I'm a committed fan of the man. Superb stuff!\",\n",
            "       b\"I gave this film 10 not because it is a superbly consistent movie, but for it's pure ability to evoke emotions in its audience. The story of one-woman's-struggle-against-all-odds is an old clich\\xc3\\xa9 by now, but very few films have carried it off with so much warmth and sincerity as The Color Purple.<br /><br />It also showed a different side to the African-American experience - showing that after slaves were granted freedom many fell into the ways of the hated 'white man' and were abusive of their own people. I find this an important point as it goes against the portray-white-on-black-violence-and-win-an-Oscar trend.<br /><br />Also the acting performances are superb - especially Oprah who I now have a new found respect for.<br /><br />Well worth watching - but keep some tissue handy.\",\n",
            "       b'I admit to being somewhat jaded about the movie genre of a young child softening the heart of his/her reluctant guardian. I\\'ve seen enough of them \\xc2\\x97 Baby Boom, Kolya, About a Boy, Mostly Martha, and to some extent, Whale Rider \\xc2\\x97 to expect to be bored by the formula. What held my attention in The King of Masks was the grimness of the setting: small-town China in the 1930\\'s. Extreme poverty was the norm, and girl children were considered so worthless to poor parents that they killed them at birth or gave them to whomever would take them on the black market. When Wang discovers his purchased grandson, whom he\\'s nicknamed \"Doggie,\" is a granddaughter, he initially casts her out, even though she\\'s showed great promise as street-performing heir. Even after he reluctantly takes her back, he\\'s not too upset when she\\'s kidnapped. The film is gritty then, showing the lengths to which a young, street-smart girl had to go to survive in that society.<br /><br />The two lead performances are believable and beguiling in their societal context. In a Western society, one would expect at least a hint of resentment from Wang at not having achieved more material success. Wang so thoroughly accepts his station as a celebrated artist with low societal status, though, that I did, too. While Doggie exhibits a level of precociousness and cunning that would be suspect in a modern, suburban child, it\\'s completely believable in the context of a kid constantly in survival mode in a society that treats poor girls like garbage. And after learning that her previous seven owners have physically and mentally abused her, her fierce attachment to Wang makes perfect sense.<br /><br />The peek at small-town life in a foreign country, the naturalness of the two lead actors, the surprising plot twists, and of course, the heartwarming resolution all contribute to a very watchable film.',\n",
            "       ...,\n",
            "       b'<br /><br />Superb film with no actual spoken dialogue which enhances the level of suspense. The whole approach gives a completely different twist to a war film.<br /><br />Well worth watching again if only it could be found. I saw it perhaps 20 or so years ago. - Fantastic!',\n",
            "       b'David Beckham is a British soccer star and the husband of Victoria Beckham (\"Posh Spice\" of the Spice Girls). His trademark is a goal shot that curves across the pitch and into the net. The soccer equivalent of an unhittable curve ball in baseball. \"Bend it like Beckham\" means making that type of spectacular shot. Apart from that, and a little shrine to him in the main character\\'s bedroom and a faux-cameo at the very end, the movie has nothing to do with him.<br /><br />The movie is full of little soccer in-jokes, such as the present that one of the characters\\' parents give her of a jersey with the number 9 on it (property of the great Mia Hamm, to those in the know), references to \"Posh \\'n\\' Becks,\" the video homage to the WUSA one of the characters plays for a disbelieving friend (\"They *have* that??\"), lesbian gags, sports-bra gags, and so on.<br /><br />The story is about a teenage girl in England who idolizes Beckham and wants to be a soccer star. She has a real gift, but the two seemingly insurmountable obstacles she must overcome are the absence of a professional women\\'s league in the UK (hence their fascination with our WUSA), and her parents, who are Indian immigrants set in very old-fashioned ways that do not allow daughters, among other things, to engage in contact sports. The girl\\'s family are portrayed as figures of ironic fun, but with great affection -- think My Big Fat Greek Wedding. The girl loves and respects them enough to go through sitcom hell to conceal her growing soccer stardom from them.',\n",
            "       b'Four porn stars romping through the Irish woods sounds like a film to watch. We have Ginger Lynn Allen, Chasey Lain, Taylor Hayes, and Jenna Jameson all together in one film. Are you licking your lips? Well the mutant creatures who resulted from centuries of inbreeding were certainly licking their lips as they feasted on the entrails of their victims.<br /><br />Yes, there was some flesh exposed - far too little considering the cast - but, it was soon ripped open to expose dinner for these creatures. There was definitely some action that probably has not been seen before, and more than one person lost their head in the situation.<br /><br />Unfortunately, director Christian Viel did not show much promise and I am not likely to watch his later efforts.'],\n",
            "      dtype=object)>, <tf.Tensor: shape=(20000,), dtype=int32, numpy=array([1, 1, 1, ..., 1, 1, 0], dtype=int32)>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ihhDVgKECrxC",
        "outputId": "319aa174-b08e-475f-8331-8083785894dc"
      },
      "source": [
        "train_feat"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([b\"Canadian director Vincenzo Natali took the art-house circuit by storm with the intriguing and astonishingly intelligent Cube, which is my personal favourite SF film of the 90s. It framed the basic conceit of a group of strangers trapped in a maze shaped like a giant cube, shot entirely on one set, and took this idea in fascinating directions. <br /><br />I've been eagerly awaiting Natali's follow-up, and although its taken five years for him to mount another project, I'm delighted to say it was worth the wait. Cypher is a fascinating exploration of one man's place in the world, and how through a completely logical chain of events, finds himself in a situation beyond his control.<br /><br />I don't want to reveal too much about the plot, because one of the joys of Cypher is the different avenues it takes us down. It is so refreshing in this day and age to see a SF film that has more than one idea in it's head. Cypher is such a film.<br /><br />Morgan Sullivan (Jeremy Northam), one of the blandest people to ever walk the planet, is hired by the company DigiCorp. They send him to different parts of America to record different seminars. To his bewilderment, they are unbelievably boring. Covering topics as mundane as shaving cream and cheese.<br /><br />While Morgan is waiting for one seminar, he runs into Rita Foster (an impeccably cast Lucy Liu), the definition of an ice maiden. She gives him the brush-off, but there is something to her he finds irresistible. That's not too surprising considering the dry marriage he is in. <br /><br />When Rita turns up at another one of Morgan's seminars, she tells him his life is not what it appears. And I'm not saying anything more about the plot. To do so would cheapen the impact the rest of the film has on us, as well as the tortuous path that's so much fun to follow.<br /><br />As with Cube, Natali shows quite a talent for encompassing seemingly ordinary people, taking them out of the familiar, and basically seeing what will happen when they're thrust into the unknown. And Cypher follows similar patterns. But it's not a carbon copy of Cube. It has it's own inspiration.<br /><br />Cypher is a film that has more in common with conspiracy thrillers and paranoia stories. One of the great things about Cypher is the way these themes creep into the story without your knowledge. When Morgan realises his false identity is a piece of a much larger puzzle, it's as much of a shock to us as it is to him.<br /><br />One thing that distinguishes Cypher from Cube is how much more polished it is. Where Cube was confined to a minimalist setting and a shoestring budget with a cast of unknowns, Cypher is also on a low budget, but Natali economises it as much as he can, allowing him to broaden the horizon, and launching Morgan on an amazing journey through the labyrinth of his own identity.<br /><br />Natali's direction is exceptional, with a deft hand on the reins. There are some amazing camera angles from above, such as the enormity of the DigiCorp building as a vast, robust office block in conjunction to the insignificant speck that is Morgan standing outside. All the colour appears to have been bled out of the picture, which compliments the tone of the film perfectly as a modern day film-noir.<br /><br />The acting is uniformly excellent throughout. Jeremy Northam is a sympathetic figure from his loveless marriage to questioning his own identity. His performance is excellent because it's so modulated. He literally seems to transform right before our very eyes. From a clinical, spineless wimp to a confident man who will do anything to preserve his new identity.<br /><br />David Hewlett puts in a welcome appearance who made such an impact in Cube. He resides in a secret silo that looks like it was borrowed from Men in Black. His scene is one of the best because it's an exercise in carefully calculated suspense and paranoia. He is a supposed expert in identifying double-agents, and it's a fantastic piece of writing, brilliantly acted by Hewlett. All he has to do is look at Morgan, and we're drawn into his complex mind game.<br /><br />But it's Lucy Liu who's the scene stealer here. Too often she is cast in films where her potential is not utilised to full effect. But in Cypher, she is finally given a character that fits her like a glove. Rita is an aloof, guarded femme fatale that Liu inhabits with relish. I perked up every time she appeared because she is always in control, and can reduce a room to silence by the power of her icy stare alone.<br /><br />Things come to a very gratifying end, that doesn't conclude on an ambiguous note the way Cube did. But Morgan deserves his happy ending. After he's been put through the ringer like this, I cheered for him in the final scene. It's a perfect final moment because it comes as a ray of sunshine after a gloomy 90 minutes.<br /><br />Cypher succeeds on all counts. Engaging, shocking, always entertaining, it's everything that Total Recall wanted to be but wasn't. And it comes as a refreshing antidote to the overwhelming and inexplicable Matrix.<br /><br />A fine follow-up from Natali. And now I'm a committed fan of the man. Superb stuff!\",\n",
              "       b\"I gave this film 10 not because it is a superbly consistent movie, but for it's pure ability to evoke emotions in its audience. The story of one-woman's-struggle-against-all-odds is an old clich\\xc3\\xa9 by now, but very few films have carried it off with so much warmth and sincerity as The Color Purple.<br /><br />It also showed a different side to the African-American experience - showing that after slaves were granted freedom many fell into the ways of the hated 'white man' and were abusive of their own people. I find this an important point as it goes against the portray-white-on-black-violence-and-win-an-Oscar trend.<br /><br />Also the acting performances are superb - especially Oprah who I now have a new found respect for.<br /><br />Well worth watching - but keep some tissue handy.\",\n",
              "       b'I admit to being somewhat jaded about the movie genre of a young child softening the heart of his/her reluctant guardian. I\\'ve seen enough of them \\xc2\\x97 Baby Boom, Kolya, About a Boy, Mostly Martha, and to some extent, Whale Rider \\xc2\\x97 to expect to be bored by the formula. What held my attention in The King of Masks was the grimness of the setting: small-town China in the 1930\\'s. Extreme poverty was the norm, and girl children were considered so worthless to poor parents that they killed them at birth or gave them to whomever would take them on the black market. When Wang discovers his purchased grandson, whom he\\'s nicknamed \"Doggie,\" is a granddaughter, he initially casts her out, even though she\\'s showed great promise as street-performing heir. Even after he reluctantly takes her back, he\\'s not too upset when she\\'s kidnapped. The film is gritty then, showing the lengths to which a young, street-smart girl had to go to survive in that society.<br /><br />The two lead performances are believable and beguiling in their societal context. In a Western society, one would expect at least a hint of resentment from Wang at not having achieved more material success. Wang so thoroughly accepts his station as a celebrated artist with low societal status, though, that I did, too. While Doggie exhibits a level of precociousness and cunning that would be suspect in a modern, suburban child, it\\'s completely believable in the context of a kid constantly in survival mode in a society that treats poor girls like garbage. And after learning that her previous seven owners have physically and mentally abused her, her fierce attachment to Wang makes perfect sense.<br /><br />The peek at small-town life in a foreign country, the naturalness of the two lead actors, the surprising plot twists, and of course, the heartwarming resolution all contribute to a very watchable film.',\n",
              "       ...,\n",
              "       b'<br /><br />Superb film with no actual spoken dialogue which enhances the level of suspense. The whole approach gives a completely different twist to a war film.<br /><br />Well worth watching again if only it could be found. I saw it perhaps 20 or so years ago. - Fantastic!',\n",
              "       b'David Beckham is a British soccer star and the husband of Victoria Beckham (\"Posh Spice\" of the Spice Girls). His trademark is a goal shot that curves across the pitch and into the net. The soccer equivalent of an unhittable curve ball in baseball. \"Bend it like Beckham\" means making that type of spectacular shot. Apart from that, and a little shrine to him in the main character\\'s bedroom and a faux-cameo at the very end, the movie has nothing to do with him.<br /><br />The movie is full of little soccer in-jokes, such as the present that one of the characters\\' parents give her of a jersey with the number 9 on it (property of the great Mia Hamm, to those in the know), references to \"Posh \\'n\\' Becks,\" the video homage to the WUSA one of the characters plays for a disbelieving friend (\"They *have* that??\"), lesbian gags, sports-bra gags, and so on.<br /><br />The story is about a teenage girl in England who idolizes Beckham and wants to be a soccer star. She has a real gift, but the two seemingly insurmountable obstacles she must overcome are the absence of a professional women\\'s league in the UK (hence their fascination with our WUSA), and her parents, who are Indian immigrants set in very old-fashioned ways that do not allow daughters, among other things, to engage in contact sports. The girl\\'s family are portrayed as figures of ironic fun, but with great affection -- think My Big Fat Greek Wedding. The girl loves and respects them enough to go through sitcom hell to conceal her growing soccer stardom from them.',\n",
              "       b'Four porn stars romping through the Irish woods sounds like a film to watch. We have Ginger Lynn Allen, Chasey Lain, Taylor Hayes, and Jenna Jameson all together in one film. Are you licking your lips? Well the mutant creatures who resulted from centuries of inbreeding were certainly licking their lips as they feasted on the entrails of their victims.<br /><br />Yes, there was some flesh exposed - far too little considering the cast - but, it was soon ripped open to expose dinner for these creatures. There was definitely some action that probably has not been seen before, and more than one person lost their head in the situation.<br /><br />Unfortunately, director Christian Viel did not show much promise and I am not likely to watch his later efforts.'],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DkXKL3czDDRy",
        "outputId": "0ea4a57d-b14f-405a-9ae9-9ebb4accbcdd"
      },
      "source": [
        "train_lab"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1, 1, 1, ..., 1, 1, 0], dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "sVPEW9utDHxR",
        "outputId": "9c3a372a-2599-4a44-b653-6a6f89b43d15"
      },
      "source": [
        "train = pd.DataFrame([train_feat, train_lab]).T\r\n",
        "train.columns = ['DATA_COLUMN', 'LABEL_COLUMN']\r\n",
        "train['DATA_COLUMN'] = train['DATA_COLUMN'].str.decode(\"utf-8\")\r\n",
        "train.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DATA_COLUMN</th>\n",
              "      <th>LABEL_COLUMN</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Canadian director Vincenzo Natali took the art...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>I gave this film 10 not because it is a superb...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I admit to being somewhat jaded about the movi...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>For a long time, 'The Menagerie' was my favori...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>A truly frightening film. Feels as if it were ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                         DATA_COLUMN LABEL_COLUMN\n",
              "0  Canadian director Vincenzo Natali took the art...            1\n",
              "1  I gave this film 10 not because it is a superb...            1\n",
              "2  I admit to being somewhat jaded about the movi...            1\n",
              "3  For a long time, 'The Menagerie' was my favori...            1\n",
              "4  A truly frightening film. Feels as if it were ...            0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kTetTzqSEGZc"
      },
      "source": [
        "I will do the same operations for the test dataset with the following lines:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "qgJ1TKHvDfcb",
        "outputId": "9c4f2801-8465-41ff-a885-f312805459de"
      },
      "source": [
        "for j in test.take(1):\r\n",
        "  test_feat = j[0].numpy()\r\n",
        "  test_lab = j[1].numpy()\r\n",
        "\r\n",
        "test = pd.DataFrame([test_feat, test_lab]).T\r\n",
        "test.columns = ['DATA_COLUMN', 'LABEL_COLUMN']\r\n",
        "test['DATA_COLUMN'] = test['DATA_COLUMN'].str.decode(\"utf-8\")\r\n",
        "test.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DATA_COLUMN</th>\n",
              "      <th>LABEL_COLUMN</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>I can't believe that so much talent can be was...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>This movie blows - let's get that straight rig...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>The saddest thing about this \"tribute\" is that...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>I'm only rating this film as a 3 out of pity b...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Something surprised me about this movie - it w...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                         DATA_COLUMN LABEL_COLUMN\n",
              "0  I can't believe that so much talent can be was...            0\n",
              "1  This movie blows - let's get that straight rig...            0\n",
              "2  The saddest thing about this \"tribute\" is that...            0\n",
              "3  I'm only rating this film as a 3 out of pity b...            0\n",
              "4  Something surprised me about this movie - it w...            1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bfTBwuk1EiXN"
      },
      "source": [
        "## Creating Input Sequences"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kcGD79J-EiLd"
      },
      "source": [
        "We have two pandas Dataframe objects waiting for us to convert them into suitable objects for the BERT model. We will take advantage of the InputExample function that helps us to create sequences from our dataset. The *InputExample* function can be called as follows:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6Vr3xwtRD10T",
        "outputId": "5104137a-fe0f-4c7b-c8ff-a19d50a857ff"
      },
      "source": [
        "InputExample(guid=None,\r\n",
        "             text_a = \"Hello, world\",\r\n",
        "             text_b = None,\r\n",
        "             label = 1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "InputExample(guid=None, text_a='Hello, world', text_b=None, label=1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AZwlq-yLE1un"
      },
      "source": [
        "## Now we will create two main functions:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hjDw_QteE_TY"
      },
      "source": [
        "- 1 — convert_data_to_examples: This will accept our train and test datasets and convert each row into an InputExample object."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pg3ebLyoE_Od"
      },
      "source": [
        "- 2 — convert_examples_to_tf_dataset: This function will tokenize the InputExample objects, then create the required input format with the tokenized objects, finally, create an input dataset that we can feed to the model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RNtn6nryEvWJ"
      },
      "source": [
        "def convert_data_to_examples(train, test, DATA_COLUMN, LABEL_COLUMN): \r\n",
        "  train_InputExamples = train.apply(lambda x: InputExample(guid=None, # Globally unique ID for bookkeeping, unused in this case\r\n",
        "                                                          text_a = x[DATA_COLUMN], \r\n",
        "                                                          text_b = None,\r\n",
        "                                                          label = x[LABEL_COLUMN]), axis = 1)\r\n",
        "\r\n",
        "  validation_InputExamples = test.apply(lambda x: InputExample(guid=None, # Globally unique ID for bookkeeping, unused in this case\r\n",
        "                                                          text_a = x[DATA_COLUMN], \r\n",
        "                                                          text_b = None,\r\n",
        "                                                          label = x[LABEL_COLUMN]), axis = 1)\r\n",
        "  \r\n",
        "  return train_InputExamples, validation_InputExamples\r\n",
        "\r\n",
        "  train_InputExamples, validation_InputExamples = convert_data_to_examples(train, \r\n",
        "                                                                           test, \r\n",
        "                                                                           'DATA_COLUMN', \r\n",
        "                                                                           'LABEL_COLUMN')\r\n",
        "  \r\n",
        "def convert_examples_to_tf_dataset(examples, tokenizer, max_length=128):\r\n",
        "    features = [] # -> will hold InputFeatures to be converted later\r\n",
        "\r\n",
        "    for e in examples:\r\n",
        "        # Documentation is really strong for this method, so please take a look at it\r\n",
        "        input_dict = tokenizer.encode_plus(\r\n",
        "            e.text_a,\r\n",
        "            add_special_tokens=True,\r\n",
        "            max_length=max_length, # truncates if len(s) > max_length\r\n",
        "            return_token_type_ids=True,\r\n",
        "            return_attention_mask=True,\r\n",
        "            pad_to_max_length=True, # pads to the right by default # CHECK THIS for pad_to_max_length\r\n",
        "            truncation=True\r\n",
        "        )\r\n",
        "\r\n",
        "        input_ids, token_type_ids, attention_mask = (input_dict[\"input_ids\"],\r\n",
        "            input_dict[\"token_type_ids\"], input_dict['attention_mask'])\r\n",
        "\r\n",
        "        features.append(\r\n",
        "            InputFeatures(\r\n",
        "                input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, label=e.label\r\n",
        "            )\r\n",
        "        )\r\n",
        "\r\n",
        "    def gen():\r\n",
        "        for f in features:\r\n",
        "            yield (\r\n",
        "                {\r\n",
        "                    \"input_ids\": f.input_ids,\r\n",
        "                    \"attention_mask\": f.attention_mask,\r\n",
        "                    \"token_type_ids\": f.token_type_ids,\r\n",
        "                },\r\n",
        "                f.label,\r\n",
        "            )\r\n",
        "\r\n",
        "    return tf.data.Dataset.from_generator(\r\n",
        "        gen,\r\n",
        "        ({\"input_ids\": tf.int32, \"attention_mask\": tf.int32, \"token_type_ids\": tf.int32}, tf.int64),\r\n",
        "        (\r\n",
        "            {\r\n",
        "                \"input_ids\": tf.TensorShape([None]),\r\n",
        "                \"attention_mask\": tf.TensorShape([None]),\r\n",
        "                \"token_type_ids\": tf.TensorShape([None]),\r\n",
        "            },\r\n",
        "            tf.TensorShape([]),\r\n",
        "        ),\r\n",
        "    )\r\n",
        "\r\n",
        "\r\n",
        "DATA_COLUMN = 'DATA_COLUMN'\r\n",
        "LABEL_COLUMN = 'LABEL_COLUMN'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PiC4aD02IfhR"
      },
      "source": [
        "## We can call the functions we created above with the following lines:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BXrJUI4lIirr",
        "outputId": "bbf3b640-d4cc-4e66-a445-b1f47c0505db"
      },
      "source": [
        "train_InputExamples, validation_InputExamples = convert_data_to_examples(train, test, DATA_COLUMN, LABEL_COLUMN)\r\n",
        "\r\n",
        "train_data = convert_examples_to_tf_dataset(list(train_InputExamples), tokenizer)\r\n",
        "train_data = train_data.shuffle(100).batch(32).repeat(2)\r\n",
        "\r\n",
        "validation_data = convert_examples_to_tf_dataset(list(validation_InputExamples), tokenizer)\r\n",
        "validation_data = validation_data.batch(32)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2155: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cTzvAZUkHmeT"
      },
      "source": [
        "Our dataset containing processed input sequences are ready to be fed to the model."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MBcq0DBNHqNP"
      },
      "source": [
        "### Configuring the BERT model and Fine-tuning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ePYUF4TcHqHA"
      },
      "source": [
        "We will use Adam as our optimizer, CategoricalCrossentropy as our loss function, and SparseCategoricalAccuracy as our accuracy metric. Fine-tuning the model for 2 epochs will give us around 95% accuracy, which is great.\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ugM9xJelHPsD",
        "outputId": "d7b5cf8d-1c13-4164-fb7a-84e0254d5f01"
      },
      "source": [
        "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0), \r\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), \r\n",
        "              metrics=[tf.keras.metrics.SparseCategoricalAccuracy('accuracy')])\r\n",
        "\r\n",
        "model.fit(train_data, epochs=2, validation_data=validation_data)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/2\n",
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "WARNING:tensorflow:AutoGraph could not transform <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7f558c1c2ec0>> and will run it as-is.\n",
            "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
            "Cause: module, class, method, function, traceback, frame, or code object was expected, got cython_function_or_method\n",
            "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
            "WARNING: AutoGraph could not transform <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7f558c1c2ec0>> and will run it as-is.\n",
            "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
            "Cause: module, class, method, function, traceback, frame, or code object was expected, got cython_function_or_method\n",
            "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
            "WARNING:tensorflow:AutoGraph could not transform <function wrap at 0x7f55a7a74c20> and will run it as-is.\n",
            "Cause: while/else statement not yet supported\n",
            "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
            "WARNING: AutoGraph could not transform <function wrap at 0x7f55a7a74c20> and will run it as-is.\n",
            "Cause: while/else statement not yet supported\n",
            "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
            "   1250/Unknown - 1980s 2s/step - loss: 0.3540 - accuracy: 0.8399WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
            "1250/1250 [==============================] - 2060s 2s/step - loss: 0.3540 - accuracy: 0.8399 - val_loss: 0.3287 - val_accuracy: 0.8860\n",
            "Epoch 2/2\n",
            "1250/1250 [==============================] - 2004s 2s/step - loss: 0.0993 - accuracy: 0.9657 - val_loss: 0.4488 - val_accuracy: 0.8762\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f54ac0a3650>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zzO24avYJ-5K"
      },
      "source": [
        "Training the model might take a while, so ensure you enabled the GPU acceleration from the Notebook Settings. After our training is completed, we can move onto making sentiment predictions."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        },
        "id": "BJh468HzZL6r",
        "outputId": "820bfbc6-1ef4-4e8d-ee20-9a58dfb37ea5"
      },
      "source": [
        "model.save(\"BERT_With_TF.h5\")  #---- This .h5 doesn't work to save the subclassed model its because of not sequential model.\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NotImplementedError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNotImplementedError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-200-53560a58c3f7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"BERT_With_TF.h5\"\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m#---- This .h5 doesn't work to save the subclassed model its because of not sequential model.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, filepath, overwrite, include_optimizer, save_format, signatures, options, save_traces)\u001b[0m\n\u001b[1;32m   2000\u001b[0m     \u001b[0;31m# pylint: enable=line-too-long\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2001\u001b[0m     save.save_model(self, filepath, overwrite, include_optimizer, save_format,\n\u001b[0;32m-> 2002\u001b[0;31m                     signatures, options, save_traces)\n\u001b[0m\u001b[1;32m   2003\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2004\u001b[0m   def save_weights(self,\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/saving/save.py\u001b[0m in \u001b[0;36msave_model\u001b[0;34m(model, filepath, overwrite, include_optimizer, save_format, signatures, options, save_traces)\u001b[0m\n\u001b[1;32m    145\u001b[0m         not isinstance(model, sequential.Sequential)):\n\u001b[1;32m    146\u001b[0m       raise NotImplementedError(\n\u001b[0;32m--> 147\u001b[0;31m           \u001b[0;34m'Saving the model to HDF5 format requires the model to be a '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    148\u001b[0m           \u001b[0;34m'Functional model or a Sequential model. It does not work for '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m           \u001b[0;34m'subclassed models, because such models are defined via the body of '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNotImplementedError\u001b[0m: Saving the model to HDF5 format requires the model to be a Functional model or a Sequential model. It does not work for subclassed models, because such models are defined via the body of a Python method, which isn't safely serializable. Consider saving to the Tensorflow SavedModel format (by setting save_format=\"tf\") or using `save_weights`."
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z2cYEfo-ZohZ"
      },
      "source": [
        "# from tensorflow.keras.models import load_model\r\n",
        "# # my_model=load_model()\r\n",
        "# # my_model = tf.saved_model.load(\"/content/Bert model/saved_model.pb\")\r\n",
        "\r\n",
        "# import tensorflow as tf\r\n",
        "# converter = tf.lite.TFLiteConverter.from_keras_model(\"/content/Bert model/saved_model.pb\")\r\n",
        "# tflite_model = converter.convert()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E0yh8HcUbNnX",
        "outputId": "ab5c3e83-c73f-4e63-8b3d-0a69c3e7f371"
      },
      "source": [
        "model.save('/content/drive/MyDrive/Colab Notebooks/BERT_Model', save_format='tf')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
            "WARNING:absl:Found untraced functions such as embeddings_layer_call_and_return_conditional_losses, embeddings_layer_call_fn, encoder_layer_call_and_return_conditional_losses, encoder_layer_call_fn, pooler_layer_call_and_return_conditional_losses while saving (showing 5 of 1070). These functions will not be directly callable after loading.\n",
            "WARNING:absl:Found untraced functions such as embeddings_layer_call_and_return_conditional_losses, embeddings_layer_call_fn, encoder_layer_call_and_return_conditional_losses, encoder_layer_call_fn, pooler_layer_call_and_return_conditional_losses while saving (showing 5 of 1070). These functions will not be directly callable after loading.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/Colab Notebooks/BERT_Model/assets\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/Colab Notebooks/BERT_Model/assets\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7IeCRzbHaZWg",
        "outputId": "5626d490-136b-4251-d1ff-65125c93652e"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sovl4mu7KIbo"
      },
      "source": [
        "## Making Predictions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q54_-duuKILp"
      },
      "source": [
        "I created a list of two reviews I created. The first one is a positive review, while the second one is clearly negative."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fzNMGdneHPxd"
      },
      "source": [
        "pred_sentences = ['This was an awesome movie. I watch it twice my time watching this beautiful movie if I have known it was this good',\r\n",
        "                  'One of the worst movies of all time. I cannot believe I wasted two hours of my life for this movie']\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZfRnuJ0cndLV",
        "outputId": "bdf475eb-2b86-4d60-b2b7-76bc3a9fd8a0"
      },
      "source": [
        "type(test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "list"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YtoE22MNQ57m"
      },
      "source": [
        "We need to tokenize our reviews with our pre-trained BERT tokenizer. We will then feed these tokenized sequences to our model and run a final softmax layer to get the predictions. We can then use the *argmax* function to determine whether our sentiment prediction for the review is positive or negative. Finally, we will print out the results with a simple for loop. The following lines do all of these said operations:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nRB8Dywhi0NW",
        "outputId": "3ba8c8de-57e3-4d13-802a-7b08ef45c17f"
      },
      "source": [
        "tf_batch = tokenizer(pred_sentences, max_length=128, padding=True, truncation=True, return_tensors='tf')\r\n",
        "tf_outputs = model(tf_batch)\r\n",
        "tf_outputs"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TFSequenceClassifierOutput([('logits',\n",
              "                             <tf.Tensor: shape=(2, 2), dtype=float32, numpy=\n",
              "                             array([[-3.907434 ,  3.241432 ],\n",
              "                                    [ 3.935399 , -3.3383203]], dtype=float32)>)])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "odk54y3NjUdZ",
        "outputId": "6983057b-56ac-4292-ea3e-2081060d4578"
      },
      "source": [
        "print(tf_outputs[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tf.Tensor(\n",
            "[[-3.907434   3.241432 ]\n",
            " [ 3.935399  -3.3383203]], shape=(2, 2), dtype=float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FV8QJ61AjnBI",
        "outputId": "6e382d1b-f871-432e-b642-77631bdc6d0f"
      },
      "source": [
        "\r\n",
        "tf_predictions = tf.nn.softmax(tf_outputs[0], axis=1)\r\n",
        "labels = ['Negative','Positive']\r\n",
        "label = tf.argmax(tf_predictions, axis=-1)\r\n",
        "label = label.numpy()\r\n",
        "for i in range(len(pred_sentences)):\r\n",
        "  print(pred_sentences[i], \": \\n\", labels[label[i]])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "This was an awesome movie. I watch it twice my time watching this beautiful movie if I have known it was this good : \n",
            " Positive\n",
            "One of the worst movies of all time. I cannot believe I wasted two hours of my life for this movie : \n",
            " Negative\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "076OISRJjR7Q"
      },
      "source": [
        "d=test[\"DATA_COLUMN\"].astype(str)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LpnE3os2qvp1",
        "outputId": "13cb6b09-b587-4a00-ec1b-ae33401d76ac"
      },
      "source": [
        "d.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5000,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 87
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rge_qDH0pM1j"
      },
      "source": [
        "l=[]\r\n",
        "for i in d:\r\n",
        "  l.append(i)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        },
        "id": "tDbTf2Bgrj_0",
        "outputId": "7074c6f3-7941-4d08-fdb8-d2f0d5fed256"
      },
      "source": [
        "test1=l[0:820]\r\n",
        "tf_batch1 = tokenizer(test1, max_length=128, padding=True, truncation=True, return_tensors='tf')\r\n",
        "tf_outputs1 = model(tf_batch1)\r\n",
        "tf_predictions1 = tf.nn.softmax(tf_outputs1[0], axis=-1)\r\n",
        "labels = ['Negative','Positive']\r\n",
        "label1 = tf.argmax(tf_predictions1, axis=1)\r\n",
        "label1 = label1.numpy()\r\n",
        "# for i in range(len(test1)):\r\n",
        "#   print(test[i], \": \\n\", labels[label1[i]])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ResourceExhaustedError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-155-35a8dcff756e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mtest1\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m820\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mtf_batch1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_length\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m128\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpadding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtruncation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_tensors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'tf'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mtf_outputs1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf_batch1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mtf_predictions1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf_outputs1\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'Negative'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'Positive'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1010\u001b[0m         with autocast_variable.enable_auto_cast_variables(\n\u001b[1;32m   1011\u001b[0m             self._compute_dtype_object):\n\u001b[0;32m-> 1012\u001b[0;31m           \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1013\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1014\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_activity_regularizer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_tf_bert.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, output_attentions, output_hidden_states, return_dict, labels, training, **kwargs)\u001b[0m\n\u001b[1;32m   1544\u001b[0m             \u001b[0moutput_hidden_states\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"output_hidden_states\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1545\u001b[0m             \u001b[0mreturn_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"return_dict\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1546\u001b[0;31m             \u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"training\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1547\u001b[0m         )\n\u001b[1;32m   1548\u001b[0m         \u001b[0mpooled_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1010\u001b[0m         with autocast_variable.enable_auto_cast_variables(\n\u001b[1;32m   1011\u001b[0m             self._compute_dtype_object):\n\u001b[0;32m-> 1012\u001b[0;31m           \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1013\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1014\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_activity_regularizer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_tf_bert.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, output_attentions, output_hidden_states, return_dict, training, **kwargs)\u001b[0m\n\u001b[1;32m    799\u001b[0m             \u001b[0moutput_hidden_states\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"output_hidden_states\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    800\u001b[0m             \u001b[0mreturn_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"return_dict\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 801\u001b[0;31m             \u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"training\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    802\u001b[0m         )\n\u001b[1;32m    803\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1010\u001b[0m         with autocast_variable.enable_auto_cast_variables(\n\u001b[1;32m   1011\u001b[0m             self._compute_dtype_object):\n\u001b[0;32m-> 1012\u001b[0;31m           \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1013\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1014\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_activity_regularizer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_tf_bert.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, hidden_states, attention_mask, head_mask, output_attentions, output_hidden_states, return_dict, training)\u001b[0m\n\u001b[1;32m    550\u001b[0m                 \u001b[0mhead_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhead_mask\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    551\u001b[0m                 \u001b[0moutput_attentions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_attentions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 552\u001b[0;31m                 \u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    553\u001b[0m             )\n\u001b[1;32m    554\u001b[0m             \u001b[0mhidden_states\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlayer_outputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1010\u001b[0m         with autocast_variable.enable_auto_cast_variables(\n\u001b[1;32m   1011\u001b[0m             self._compute_dtype_object):\n\u001b[0;32m-> 1012\u001b[0;31m           \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1013\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1014\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_activity_regularizer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_tf_bert.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, hidden_states, attention_mask, head_mask, output_attentions, training)\u001b[0m\n\u001b[1;32m    513\u001b[0m         )\n\u001b[1;32m    514\u001b[0m         \u001b[0mattention_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mattention_outputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 515\u001b[0;31m         \u001b[0mintermediate_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mintermediate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden_states\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattention_output\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    516\u001b[0m         layer_output = self.bert_output(\n\u001b[1;32m    517\u001b[0m             \u001b[0mhidden_states\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mintermediate_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_tensor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattention_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1010\u001b[0m         with autocast_variable.enable_auto_cast_variables(\n\u001b[1;32m   1011\u001b[0m             self._compute_dtype_object):\n\u001b[0;32m-> 1012\u001b[0;31m           \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1013\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1014\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_activity_regularizer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_tf_bert.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, hidden_states)\u001b[0m\n\u001b[1;32m    461\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    462\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden_states\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 463\u001b[0;31m         \u001b[0mhidden_states\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdense\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhidden_states\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    464\u001b[0m         \u001b[0mhidden_states\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mintermediate_act_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden_states\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    465\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1010\u001b[0m         with autocast_variable.enable_auto_cast_variables(\n\u001b[1;32m   1011\u001b[0m             self._compute_dtype_object):\n\u001b[0;32m-> 1012\u001b[0;31m           \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1013\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1014\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_activity_regularizer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/layers/einsum_dense.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m    201\u001b[0m     \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspecial_math_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meinsum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mequation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkernel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    202\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 203\u001b[0;31m       \u001b[0mret\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    204\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactivation\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m       \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactivation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mret\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/math_ops.py\u001b[0m in \u001b[0;36mbinary_op_wrapper\u001b[0;34m(x, y)\u001b[0m\n\u001b[1;32m   1162\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1163\u001b[0m       \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1164\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1165\u001b[0m       \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mTypeError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1166\u001b[0m         \u001b[0;31m# Even if dispatching the op failed, the RHS may be a tensor aware\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/dispatch.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    199\u001b[0m     \u001b[0;34m\"\"\"Call target, and fall back on dispatchers if there is a TypeError.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 201\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    202\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mTypeError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m       \u001b[0;31m# Note: convert_to_eager_tensor currently raises a ValueError, not a\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/math_ops.py\u001b[0m in \u001b[0;36m_add_dispatch\u001b[0;34m(x, y, name)\u001b[0m\n\u001b[1;32m   1484\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mgen_math_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1485\u001b[0m   \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1486\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgen_math_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_v2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1487\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1488\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/gen_math_ops.py\u001b[0m in \u001b[0;36madd_v2\u001b[0;34m(x, y, name)\u001b[0m\n\u001b[1;32m    470\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    471\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0m_core\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 472\u001b[0;31m       \u001b[0m_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_from_not_ok_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    473\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0m_core\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_FallbackException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    474\u001b[0m       \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36mraise_from_not_ok_status\u001b[0;34m(e, name)\u001b[0m\n\u001b[1;32m   6860\u001b[0m   \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m\" name: \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m\"\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6861\u001b[0m   \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 6862\u001b[0;31m   \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_status_to_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   6863\u001b[0m   \u001b[0;31m# pylint: enable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6864\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/six.py\u001b[0m in \u001b[0;36mraise_from\u001b[0;34m(value, from_value)\u001b[0m\n",
            "\u001b[0;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[820,128,3072] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc [Op:AddV2]"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XXWhYfSV2T4K"
      },
      "source": [
        "# OOM when allocating tensor with shape[820,128,3072] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc [Op:AddV2]\r\n",
        "\r\n",
        "\r\n",
        "# ERROR : OUT OF MEMORY \r\n",
        "\r\n",
        "## So thats why only 810 records are testified"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j6LexgkupeFK",
        "outputId": "10cacc34-163b-4e82-e234-ed83103ee099"
      },
      "source": [
        "df1=label1\r\n",
        "df1.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(810,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 153
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SnJbdhl_s0aY"
      },
      "source": [
        "import numpy as np\r\n",
        "y_test= test[\"LABEL_COLUMN\"].iloc[0:810]\r\n",
        "y_pred=df1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8Qi9Ahdux10m",
        "outputId": "c75e4c07-16f7-4a48-caf4-5262526a68ec"
      },
      "source": [
        "y_test1=list(y_test.astype(int))\r\n",
        "y_pred1=list(y_pred)\r\n",
        "print(y_pred1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RlQ8tGGPqRIY"
      },
      "source": [
        "from sklearn.metrics import confusion_matrix\r\n",
        "from sklearn.metrics import accuracy_score\r\n",
        "from sklearn.metrics import classification_report\r\n",
        "from mlxtend.evaluate import confusion_matrix\r\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WUkO-H_iw-JP",
        "outputId": "9e17c3af-925a-4f42-ed57-1a3bd59cfb42"
      },
      "source": [
        "cm=confusion_matrix(y_test1,y_pred1)\r\n",
        "cm"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[322,  68],\n",
              "       [ 28, 392]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 188
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vcqwrmGaxmQN",
        "outputId": "a1222036-059d-4007-aac9-d61e76ce0d2d"
      },
      "source": [
        "from sklearn.metrics import accuracy_score\r\n",
        "print(f\"BERT MODEL- Accuracy_Score: {accuracy_score(y_test1,y_pred1)}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "BERT MODEL- Accuracy_Score: 0.8814814814814815\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-5tygnw7xFk-",
        "outputId": "a968ce44-c14a-436a-af15-9f746952cede"
      },
      "source": [
        "print(classification_report(y_test1,y_pred1,target_names=['Negative', 'Positive']))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "    Negative       0.92      0.83      0.87       390\n",
            "    Positive       0.85      0.93      0.89       420\n",
            "\n",
            "    accuracy                           0.88       810\n",
            "   macro avg       0.89      0.88      0.88       810\n",
            "weighted avg       0.88      0.88      0.88       810\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gUgtLN2Wvilx"
      },
      "source": [
        "#plotting confusion matrix\r\n",
        "def plot_confusion_matrix(cm, classes,\r\n",
        "                          normalize=False,\r\n",
        "                          title='BERT_TF Confusion matrix',\r\n",
        "                          cmap=plt.cm.YlOrRd):\r\n",
        "    \"\"\"\r\n",
        "    See full source and example: \r\n",
        "    http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\r\n",
        "    \r\n",
        "    This function prints and plots the confusion matrix.\r\n",
        "    Normalization can be applied by setting `normalize=True`.\r\n",
        "    \"\"\"\r\n",
        "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\r\n",
        "    plt.title(title,fontsize=17)\r\n",
        "    plt.colorbar()\r\n",
        "    tick_marks = np.arange(len(classes))\r\n",
        "    plt.xticks(tick_marks, classes, rotation=0)\r\n",
        "    plt.yticks(tick_marks, classes)\r\n",
        "\r\n",
        "    if normalize:\r\n",
        "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\r\n",
        "        print(\"Normalized confusion matrix\")\r\n",
        "    else:\r\n",
        "        print('Confusion matrix, without normalization')\r\n",
        "\r\n",
        "    thresh = cm.max() / 2.\r\n",
        "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\r\n",
        "        plt.text(j, i, cm[i, j],\r\n",
        "                 horizontalalignment=\"right\",\r\n",
        "                 color=\"White\" if cm[i, j] > thresh else \"black\")\r\n",
        "\r\n",
        "    plt.tight_layout()\r\n",
        "    plt.ylabel('True label')\r\n",
        "    plt.xlabel('Predicted label')\r\n",
        "    plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 328
        },
        "id": "CLGdRWUMyaxG",
        "outputId": "99fa5110-82fe-4d5d-8fd6-1a5d238e120c"
      },
      "source": [
        "import itertools\r\n",
        "plot_confusion_matrix(cm,[\"Negative\",\"Postive\"])\r\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Confusion matrix, without normalization\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW8AAAEmCAYAAACtaxGwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5xU1d3H8c93dwEbFgQNWIJREjVqUIklRkWNisbErjG2GBI0MWI0PpY0MYkGExVLogmWiMYS62PvSlCMBRWJWB6JYgCxIIIgHX7PH/cMDOvO7izM7uxdvu/X67527pl7zzmzLL8585tzz1VEYGZm+VJT7Q6YmVnzOXibmeWQg7eZWQ45eJuZ5ZCDt5lZDjl4m5nlkIO3rXAkdZV0o6T3JYWki1ugje+luntWuu72QNJwScOr3Y88c/BeRkX/OYu3jyQ9I+noBo4f3sDxxVtdOq5nvfJFkqZJelxSv3RM3ybqKt76NvE66rfX2Pa9dE5jx6xZxu9Okr4j6SFJH0qaL+k9SXdJOmBZ/j2a6QLgQOAS4Gjg+lZos12S1FvSIEkbVrsvK5q6anegHfgt8H/pcVfgMOA6SV0jYki9Y98HTitRz8J6+7cBd5G9wfYEjgfuSwF8DFnQKTYk1T+4XvlrTfT/wwbq+gWwVgN9fbro8XDg6gbq+7SxxiR1Am4GDgCeBS4i6/e6wDeBOyUdGRE3NtHv5fEN4KGIOK8F27ie7HXObcE22oLewNnAo8B/m3HeXi3TnRWHg/fyezginirsSLoceAs4iiygFpsZEX8vs96Xi4+VdCvwKvDTiPgmsFQ9kn4HvNeM+gGIiE8bqOsHQIcm6vpPc9tKBpMF7l80EDx/L2lfoHYZ6m2OdYBpLdlARCzks2/IKzxJq0TErIiYV+2+5J3TJhWW/ig/BhZUuN7XgCnAJpWstzVJ6gGcCAwvNeqNiPsj4p6ic7pIulzSu5LmSnpd0mmSlvrbTSmbqyTtI+klSXMkjZP03aJjvicpgA7AsUWpnp5Fqai+DfQ7JA0q2l9V0vmS/pPaKaTLDqnfVv2ct6RvpWNnpXTYXZI2q3fMoHTuppL+KmmqpJmSbpO0dhm/50Lbu6d+vidphqRbJa0pqU7Seel3OlvS3ZK61avj26l8Yvq9T5R0RXFaLP1O/pZ2n6yfqpM0XtKjknaR9LSk2cB56bmlct6SfpHOXSptJumUVH5kU697ReOR9/JbQ1LX9LgLcCSwBVmQqq+m6Nhi8yNiemONSFqDLJUxbnk6W0GdGngtsyJiViPn7EsWOK8tp4GUYnmc7Pf5V7JPHvsAfyRLJf2k3ilfBb4F/IUspfMD4HpJL6U3vxFkKaJhwEhgaDrvw1RfuS4Hjkg/XwFWJ0sfbE+W7ir1eo4AbgD+DfwynXcS8LSkr0ZE/X/bv5OllH5F9qZ9EjA/tV2OC4DpwO+ATYEfkw3YpgIbkwXSL5H9rV4CfLfo3O+TfXL4E/BRen39gS2Br6dj7gC6AwNYOn1YnKrbiCz9dw1ZoP+gRF8Hk6XNhkp6OiI+kLR56uOtEXFDma95xRER3pZhA74HRAPbQrKUQP3jh5c4PoCnio7rmcrOI8uhr0MWlB5M5T8t0Z/xwKMVem3DgXGNPF/qdQxuot6L0nFbl9mPn6Tjjy8qE3BLKv9yvT4tALYoKluXLOf8x3r1LgCurVfWN9XRt8TrHVS0/zHw5zL/Pnqm/Q7AZOBNYLWi47ZKfzO3FJUNSufeXK/Oi1Pf1yiz7RFATVH5P4BFwBMNlM+v169VGqj3qFTvTg209fUSf5MBHFTib2x4vbKNgZnA3UBH4EXgXWDtSvxdt7fNI+/ldwrZ6AuyYPtt4HeSZkbEJfWOfRc4toE6Gsq/npW2grnAuWQjpLbgAbJgXGx8E+esnn5+UmYb+5EFymsKBRERkv4IHEo2UhtbdPyIiHil6Nj3Jb0OfKHM9so1Ddhe0gYRMaHMc7YFPgf8LCJmFvVxjKQHgX0k1UTEoqJzLq9Xxz+Bk4ENyUbvTbmyXn3/IvtC/ZoS5RuSfboh0icoSQI6kwXTkUWvZSTlmQzcWc6BEfEfSaeSfcoaAWwN7BsRH5XZ1grFwXv5jYqiLyyBmyV1BgZLujEiPix6bnZEPFpmvdeSfcTuBOwAnAF0ijREaQPebcZrKSgE7c5lHt+T7BPA/Hrlr6afG9Urf6eBOj4mS2dV0s/IZpO8I2kM8AjZKPmFRs7pmX6+3sBzr5KllLqRpUkK6r+ej9PPcl9P/dkf05ooX6tQIGlT4HyymTmr1Du+yemgRd5uzt9sRAyVdBiwB3BVRDzQjLZWKP7CsmU8BqwEbLccdfwnIh6NiPsi4lfAmcBpkr5dkR5WRyEXulUL1V9qdofKOLfBACPpMzNfIuIOsjeOH5K9pu8Dz0s6q/6xy2l5Xk9j5zdar6TVyUb5vYFfA/sDewL90nHNiRuzm3EskrqndgE2q//FtC3hX0zLKHyiWa2CdV5GlpYYnOM/6PvJcrYNpY4aMh7YWOkCpiKF2RlvV6hfsGRUW39U2bOhgyPig4i4OiKOANYnC3aDJHUoUf/49HPTBp7bjCzX+2EDz1XDbmTftRwTERdGxN3pU1ZDv+9KfxK8huzT5pnATsDpFa6/3chrEGjr9ks/X65UhSl1cCHZf/SDKlVva4qIScAVwO6SGvxPmab6fSvt3kOWIjiu3mGFi4furWD3xpONSHerV77UjBZJtWnmz2IRMRt4gywvvGqJ+kcB7wEnSFp8jKQtyEa099fLQ1dToR/148P/NHBs4aKs5qRSGiTpR2S/i1Mi4nzgRuAcSS31SS3XnPNefnsVzeVdm+wLy12BGyKifn5zNUlHlajnnmhiuiDZqORs4Oc0MiWtjfsfspTD+WlO711k08fWIcv77sKSKWtXkaUmrkj/gV8jmyq4H9lsj1epkIj4RNJNwIlpLvgbZIG8fl69MzBJ0p1kb85Tyb5Y+wHwQEQ0ePFPRCxIX8bdAIyUNIwlUwVnkF3V2laMJLum4DpJlwGzyH7n6zRw7Itko++z0hz0ucDjEVFqSmCDJPUim9p4b0RclYpPJPu/dH2aSukLe4o4eC+/XxU9nks2D/sMPjsTA7Kpa6XW0diSbE5uSRExS9KfyD6e7xsR9y9Df6sqIuZK2h/4DtmI+nSyIPYR8Azw7UgX6aRjdyebZXMI2Zvj+HTOhS3QvZPJpvT1Jxt93kv2ZlGczphFNvf5G2SzXTqRfQF4HvCHxiqPiJskfUoWqM8D5pFNmTsrPjvHu2oiYqqkfch+x78i6+cDwDEs/YUqEfGWpIFkX+JeTXZ17G6Uns/9Gel7hevIfrc/LKp7mqTjgIeA35ClUixR25m8YGZm5XLO28wsh5w2aefSmhWNLvQUEe+1UnfMrEKcNmnnJI0HPt/YMRFR7rxhM2sjPPJu/44EVq52J8yssjzyLsNaqo0eNX6faw9W3mq9anfBKmD8fz9kykczKvqJcROtGrPKXIJ9MnMfioh+TR/ZchyRytCjpo6bVu1R7W5YBWz1xO+q3QWrgD67/bLidc5iIcc3nmFcbBD/19DSzq3KwdvMjGxhlzxNv3PwNjNLHLzNzHJG5Csg5qmvZmYtyiNvM7OcEeUvlN4WOHibmSUeeZuZ5ZCDt5lZzniqoJlZTjl4m5nljEfeZmY55eBtZpYzvkjHzCynPPI2M8sZ57zNzHLKwdvMLId8ebyZWc44bWJmllMO3mZmOeORt5lZTjl4m5nljC/SMTPLqTyNvPPUVzOzFlPIeZezNVmXtJKk5yS9LGmspHNS+bWS3pY0Om29U7kkXSppnKQxkrZpqg2PvM3MkgqOZucCu0fETEkdgKckPZCe+5+IuK3e8fsAvdK2PXBF+tkafTUzy7dKjbwjMzPtdkhbNHLK/sB16bxngDUldW+qr2ZmKzw1YwO6ShpVtA34TH1SraTRwAfAIxHxbHrq3JQaGSKpUypbD5hQdPrEVFaS0yZmZkkzRrNTIqJPYwdExEKgt6Q1gTslbQGcBbwHdASGAmcAv2nhvpqZtW+VSpsUi4hpwBNAv4iYnFIjc4G/AdulwyYBGxSdtn4qa7SvZmYrPAG1Km9rsi6pWxpxI2llYE/g9UIeW5KAA4BX0il3A8ekWSc7ANMjYnJjbThtYmaW1Kix7xSLNH1Yd2CYpFqyQfItEXGvpMcldSN7rxgNnJCOvx/YFxgHzAKOa6oBB28zM9KXkRVaEzYixgBbN1C+e4njAzixOW04eJuZJV7P28wsh1Ru2qQNcPA2MwNQ5dImrcHB28wscfA2M8sZ0YzZJm2Ag7eZWZKjgbeDt5lZQU2OLlt08DYzI5tp4tkmZmY5VJOjvImDt5lZ4tkmZmY5pDIWLWkrHLzNzKjs2iatwcHbzCxx8DYzyxv5Ih0zs9zJrrCsdi/K5+BtZpZ4nreZWQ45521mlkNOm1irUaeObPzgLahjR1RXy/S7HuD98y5mg6uGsMrWWxHz5zPrhZeZePIvYMEC1jxsf7r99AQQLJr5KZNO+RVzXnmt2i/DmjBt+qf8YOCVvPLaRCRxzWUDWHnlDpxw6jXMmTOfurpaLr/gOLbbduNqdzW3JKdNrBXF3Hm8td93WfTpLKirY5OHb2XGI8OZdstdTPjBKQBseM0ldDn2cKZefQPzxk/grX0PZ+G0T+i8566sf+l5jNv9wCq/CmvKyWdeT789vsJtw37KvHkLmDV7Locddylnn34Q++zZm/sfHs3pZ9/E8Ht/We2u5lqOBt4O3u3Bok9nAaAOdaiujgiY8fDwxc/PeuFlOvTonj1+7sUl5c+/RIcen2vVvlrzTZ8+ixFPv861lx8PQMeOdXTsWIckPpkxOzvmk1n0+Nya1exmu+Cct7Wumhp6jbiHjl/4PB9deT2zR41e8lxdHWsdfiCTzvjNZ05b6+jDmfHIP1uxo7Ys3v7vB3Tr2pnjTvwrL7/yX7btvRGX/P5oLj7vaPY++HxO+9WNLIrg6QfPrnZXcy5fqwq22Oq1kkLShUX7p0ka1ALt/Lze/tOVbqPNW7SIN7/+TV7bbEdW2fYrdNrsi4ufWu+i3zLz6eeY9a/nlzpl1Z13oMsxhzH57MGt3VtrpgULFvHiy+P50fe/wUsjzmPVVTox+OJ7uOKaRxly3lFMGHsZQ849iv4Dr6x2V3OtMM+7nK3JuqSVJD0n6WVJYyWdk8o3kvSspHGS/iGpYyrvlPbHped7NtVGSy49Phc4SFLXFmwDYKngHRFfa+H22qxF02cw88l/0fkbuwKwzpkDqevahcln/W6p41b68qas/6fBjD9iAAunTqtGV60Z1u/RhfV7dGH7PpsAcMi3t+PFl8cz7KYnOehbXwXg0AO257kX/1PNbuafspsxlLOVYS6we0R8BegN9JO0A3A+MCQiNgE+Bvqn4/sDH6fyIem4RrVk8F4ADAVOqf+EpG6Sbpf0fNp2Kip/JL1TXSXpnULwl/S/kl5Izw1IZYOBlSWNlnRDKpuZft4s6ZtFbV4r6RBJtZL+mNodI+n4FvwdtLjatbtQs0ZnALRSJzrvtjNz3/wPXY45nM577MJ/vz8QYslHwQ7r9+DzN1zBhB+eyrxxb1er29YMn1t3TTZYb23eePNdAB4bMZbNv7QePbqvxT9HZjOFHh8xll5f8PcXy0tEWVtTIjMz7XZIWwC7A7el8mHAAenx/mmf9PweUuMZ+JbOef8ZGCPpD/XKLyF793lK0obAQ8BmwNnA4xHxe0n9WPKuBPD9iJgqaWXgeUm3R8SZkn4SEb0baPsfwGHAfemjyR7Aj1Kd0yPiq5I6ASMlPRwRS0Wy9AYxAKC7apfz19ByOnxuHTb4ywVQW4tqxLQ772PGg4+z5dQ3mTdhEps8egcA0+95kA/Ov4x1zxhI7Vprsd5FvwUgFixgXN/9q/kSrAyX/eEYjhxwOfPmLeALPdfhb38+nv333ZaTz7qOBQsWsdJKHRh68Q+q3c3ca8YXll0ljSraHxoRQ5euS7XAC8AmZLHwP8C0iFiQDpkIrJcerwdMAIiIBZKmA2sDU0p1oEWDd0R8Iuk6YCAwu+ipbwCbF72xrC5pNeDrwIHp3AclfVx0zkBJhTltGwC9gI8aaf4B4JIUoPsBIyJitqS9gK0kHZKOWyPVtVTwTv8QQwG+XNupzX6LMWfs67y5836fKf93l14NHj/xpDPhpDNbultWYb237MmoJ5ZOf319xy/xwvBzq9Sj9keAyr9KZ0pE9GnsgIhYCPSWtCZwJ7Dp8vVwaa0x2+Ri4EXgb0VlNcAOETGn+MBSnxIk9SUL+DtGxCxJw4GVGms0Iuak4/YGDgduLlQHnBQRDzX3hZhZOyZQCySSI2KapCeAHYE1JdWl0ff6wKR02CSyQelESXVkg8rGBqctmvMGICKmArewdArkYeCkwo6kQtpjJFmqgzRCXiuVr0GWzJ8laVNgh6K65kvqUKL5fwDHATsDD6ayh4AfFc6R9EVJqy7jyzOzdiS7yrLprel61C2NuEmp3j2B14AngMKn/mOBu9Lju9M+6fnHI6LRT/ytdaP7C4HiWScDgT7pC8NXgRNS+TnAXpJeAQ4F3gNmkAXeOkmvAYOBZ4rqGkqWV7+hgXYfBnYFHo2IeansKuBV4MXUzl/xfHczo8x5guWlVroDT0gaAzwPPBIR9wJnAKdKGkeW0746HX81sHYqPxVoMrfZYkErIlYrevw+sErR/hSyVEZ904G9U8J+R+CrETE3PbdPiXbOIPuFNNTufKBLveMXkU0vXGqKoZmt4CqYNomIMcDWDZS/BWzXQPkcsgFr2draiHND4BZJNcA84IdV7o+ZrUCamJ3XprSp4B0Rb9LAu5WZWUsToFoHbzOzfGmh2SYtxcHbzCxpxjzvqnPwNjNLcpTydvA2MwOcNjEzyy2nTczM8kU4bWJmlj+Sv7A0M8ujGs/zNjPLGdF6qz1VgIO3mRnOeZuZ5ZZz3mZmeeN53mZmOZWjvImDt5lZ4pG3mVneyDlvM7Pc8WwTM7M8knwzBjOzPHLO28wsh5zzNjPLGznnbWaWOyJfI+8cZXjMzFpYTZlbEyRtIOkJSa9KGivp5FQ+SNIkSaPTtm/ROWdJGifpDUl7N9WGR95mZpBWFazYyHsB8LOIeFFSZ+AFSY+k54ZExAVLNS1tDnwH+DLQA3hU0hcjYmGpBjzyNjMrqNDIOyImR8SL6fEM4DVgvUZO2R+4OSLmRsTbwDhgu6a6amZmhZF3ORt0lTSqaBtQslqpJ7A18Gwq+omkMZKukbRWKlsPmFB02kQaD/ZOm5iZAdlUk7qy0yZTIqJP01VqNeB24KcR8YmkK4DfApF+Xgh8f1m66+BtZlZQwdkmkjqQBe4bIuIOgIh4v+j5K4F70+4kYIOi09dPZSWVDN6SLiN7d2hQRAxsqvNmZrlSoUSyJAFXA69FxEVF5d0jYnLaPRB4JT2+G7hR0kVkX1j2Ap5rrI3GRt6jlrXjZma5U9nZJjsBRwP/ljQ6lf0cOEJSb7KB8XjgeICIGCvpFuBVspkqJzY20wQaCd4RMax4X9IqETFrGV+ImVnbV6GRd0Q8RfZ2UN/9jZxzLnBuuW002VVJO0p6FXg97X9F0uXlNmBmlgvNm21SdeW8z1wM7A18BBARLwO7tGSnzMyqQmVubUBZs00iYoKWXrGl0VyMmVkutZFRdTnKCd4TJH0NiDT15WSyq4XMzNqP5s3zrrpy0iYnACeSXe3zLtA77ZuZtS85ynk3OfKOiCnAka3QFzOz6lG+7qRTzmyTL0i6R9KHkj6QdJekL7RG58zMWlWORt7lvM/cCNwCdCe78udW4KaW7JSZWasTFVtVsDWU041VIuL6iFiQtr8DK7V0x8zMWl2ORt6NrW3SJT18QNKZwM1kl3QeTiNXCZmZ5VYbCczlaOwLyxfIgnXh1Rxf9FwAZ7VUp8zMWl0hbZITja1tslFrdsTMrLraTkqkHGVdYSlpC2BzinLdEXFdS3XKzKzViVxdpNNk8JZ0NtCXLHjfD+wDPAU4eJtZ+5KjtEk5XT0E2AN4LyKOA74CrNGivTIza205W1WwnLTJ7IhYJGmBpNWBD1j6dj1mZu1Djkbe5QTvUZLWBK4km4EyE/hXi/bKzKy1VfZOOi2unLVNfpwe/kXSg8DqETGmZbtlZlYF7SF4S9qmseci4sWW6ZKZWZW0k7TJhY08F8DuFe5Lm7Xy1huz1ajrq90Nq4Bz9N1qd8Eq4F3erXylajtfRpajsYt0dmvNjpiZVV1tOwjeZmYrlDZ0f8py5CjDY2bWwqTytiar0QaSnpD0qqSxkk5O5V0kPSLpzfRzrVQuSZdKGidpTGPfORY4eJuZFVTu7vELgJ9FxObADsCJkjYHzgQei4hewGNpH7Ir13ulbQBwRVMNlHMnHUk6StKv0/6GkrYrq/tmZnlSoZF3REwuzMiLiBlkN21fD9gfGJYOGwYckB7vD1wXmWeANSV1b6yNckbelwM7Akek/RnAn8s4z8wsP5p3J52ukkYVbQNKViv1BLYGngXWjYjJ6an3gHXT4/WACUWnTUxlJZXzheX2EbGNpJcAIuJjSR3LOM/MLF/KGFUnUyKiT9PVaTXgduCnEfGJiuqPiJAUy9RPyht5z5dUSza3G0ndgEXL2qCZWZtVuZw3kjqQBe4bIuKOVPx+IR2Sfn6Qyiex9JpR66eyksoJ3pcCdwLrSDqXbDnY88rrvplZXpSZ7y5vtomAq4HXIuKioqfuBo5Nj48F7ioqPyZ9x7gDML0ovdKgctY2uUHSC2TLwgo4ICJea7L3ZmZ5UtmFqXYCjgb+LWl0Kvs5MBi4RVJ/4B3gsPTc/cC+wDhgFnBcUw2UczOGDVNl9xSXRcR/y38dZmY5UKHJ0xHxFKUTLHs0cHwAJzanjXK+sLyPJTciXgnYCHgD+HJzGjIza/PK/8Ky6spJm2xZvJ+u/PlxicPNzPIrP7G7+WubRMSLkrZvic6YmVWNaF8jb0mnFu3WANtAS6zHaGZWXTmK3WWNvDsXPV5AlgO/vWW6Y2ZWRTmK3o0G73RxTueIOK2V+mNmVj35id2N3gatLiIWSNqpNTtkZlYV7eVOOsBzZPnt0ZLuBm4FPi08WXS5p5lZ+9BOgnfBSsBHZPesLMz3DsDB28zal/zE7kaD9zpppskrLAnaBcu8EpaZWZvUjqYK1gKr0fB7kYO3mbU/+YndjQbvyRHxm1briZlZtbWTnHd+XoWZWSXkKOo1Frw/s/KVmVm71V5y3hExtTU7YmZWdfmJ3c1fmMrMrH1qPxfpmJmtOCp7J50W5+BtZlbQHnLeZmYrHAdvM7MccvA2M8sbgSp0B+JW4OBtZgb+wtLMLLdylDbJz2cEM7MWldIm5Wzl1CZdI+kDSa8UlQ2SNEnS6LTtW/TcWZLGSXpD0t5N1e+Rt5kZZGmT2oqOZ68F/gRcV698SERcsFTT0ubAd4AvAz2ARyV9MSIWlqrcI28zM6DSI++IGAGUu8zI/sDNETE3It4GxgHbNXaCg7eZWYFU3gZdJY0q2gY0o5WfSBqT0iprpbL1gAlFx0xMZSU5eJuZFZQfvKdERJ+ibWiZLVwBbAz0BiYDFy5rV53zNjODtCRsy45nI+L9xc1JVwL3pt1JwAZFh66fykryyNvMDFi8qmA527K2IHUv2j2Q7B7BAHcD35HUSdJGQC/gucbq8sjbzKyggvO8Jd0E9CXLj08Ezgb6SupNdh/g8cDxABExVtItwKvAAuDExmaagIO3mdkSFUybRMQRDRRf3cjx5wLnllu/g7eZGbSf26CZma1YBLW11e5E2Ry8zcwKPPI2M8sZp03MzPJIDt5mZrlUk59LXxy827EJE97jmGPO5v33pyKJAQMO5OSTj2D06Dc44YTfM2fOPOrqarn88jPYbrstqt3dFV5tp44cN+IGajt1pKaultdue4jhgy6j5247sNcFp1PbsQPvvjCWu/v/gli4kC2/+y12OuOHIJg341Pu+9Eg3h/zRrVfRr555G1tQV1dHRdeeArbbLMpM2Z8yrbbHs2ee27P6adfytln/5B99tmJ++9/itNPv5Thw8tdmsFaysK58xi2+7HM/3QWNXV1HPfUjYx76CkOGDaY6/b4HlPfHE/fcwbS+9gDeema2/j47Ylcu+tRzJn2CZv024X9hv6Wq3c4rNovI79ylvPOz2cEa7bu3buyzTabAtC586pstllPJk36AEl88smnAEyfPpMePbpVs5tWZP6nswCo6VBHbYc6YuFCFs6bz9Q3xwPw1iMj2ezgvQCY+K+XmDPtk+zxM6NZff3PVaXP7Udll4RtaR55ryDGj3+Xl156g+2334KLL/4Ze+/9E0477RIWLVrE009fU+3uWaKaGga8cAddNtmQ5/98I5OeG0NNXS3dt92CyS+8wuaH9GP1DT4bpLfufwjjHhhRhR63M5W9GUOLqmpPJS1MtwJ6RdKtklZp5vk9JX23aL+PpEsr39N8mzlzFgcffDoXX/wzVl99Na644jaGDDmVCRPuY8iQU+nf/7fV7qIlsWgRf936AC5af1d6bLcV3b7ci9u/cyp7DzmLHzx7K3NnfEosXLTUOT37bs/W/Q/h0TMuKFGrlUX5GnlXuxezI6J3RGwBzANOaOb5PYHFwTsiRkXEwAr2L/fmz1/AwQefzpFH9uOgg3YHYNiwexc/PvTQb/Dcc2Or2UVrwNzpMxj/xLNs0m9nJj4zmmt3OZKrtj+Ud0Y8z0f/N37xcets+SW+ddXvuHn/HzN76rTqdbi9KH8976qrdvAu9iSwiaQukv433WniGUlbAUjateimnS9J6gwMBnZOZadI6ivpXkk1ksZLWrNQuaQ3Ja0rqZuk2yU9n7adqvR6W1xE0L//b9hss4049dSjFpf36NGNf/7zBQAef/x5evXaoFQV1opW6boWndboDEDdSp34wp5fY8rrb7FKty4A1HbswE5n/JBRf7kZgNU36M7hd1zGnUefvjgnbsspR8G7TeS8JdUB+wAPAucAL0XEAZJ2J7t5Z2/gNLJlEkdKWg2YA5wJnBYR+6V6+gJExCJJd5Gtl/s3SdsD70TE+5JuJLsB6FOSNgVHAzQAAAt0SURBVAQeAjZroE8DgAEAG26Yzy+CRo58meuvv58tt9yE3r2zDyjnnfdjrrzyl5x88gUsWLCQlVbqyNChv6hyTw1gte7rcMCwwdTU1qIaMfaWB3nzvuHs+YfT6bVfX1RTw6grbmL8E88AsOuvT2Tltdfkm5efDcCiBQu58qsHV/Ml5F+O5nkrIqrXuLQQ+HfafRL4GfAscHBEvJWOmUB2R+UfkwXjG4A7ImJiCtb1g/dpEbGfpK8Bv46IfpKGAK9GxJWSPgDeLepGN+BLETGzVD/79Nk8Ro26vmKv26rnnCVfkViO/ZV3eDfmVHQI3GeL7vH8HceWdWzNl85/ISL6VLL95qr2yHt2RPQuLlCJjyQRMVjSfcC+wEhJezdR97/I0jDdgAOA36XyGmCHiJizXD03s/bF87yX25PAkbB4JD0lIj6RtHFE/DsizgeeBzYFZgCdG6okso8UdwIXAa9FxEfpqYeBkwrHpbtamJnlarZJtUfeDRkEXCNpDDALKHyO+amk3YBFwFjggfR4oaSXgWuBl+rV9Q+yQP+9orKBwJ9T/XXACJo/y8XM2qX8jLyrGrwjYrUGyqaSpTnql59UvyzZvd7+8KJzRlHvXyMipgCHN7evZtbeCWp8MwYzsxzyyNvMLH9y9IWlg7eZGZCNutvGl5HlyE9PzcxaUmGqYIWusJR0jaQPJL1SVNZF0iPpiu9HJK2VyiXpUknj0tXl2zRVv4O3mVlBZS+PvxboV6/sTOCxiOgFPJb2IbvCvFfaBgBXNFW5g7eZ2WIqc2taRIwAptYr3h8Ylh4PY8nMuv2B6yLzDLCmpO6N1e+ct5kZsPhmDOXpKmlU0f7QiCjndlTrRsTk9Pg9YN30eD1gQtFxE1PZZEpw8DYzKyg/eE9Z3rVNIiIkLfPiUk6bmJkBrXQbtPcL6ZD084NUPgkoXpt5/VRWkoO3mRlksVsqa1sOd7NkyY9jgbuKyo9Js052AKYXpVca5LSJmdlilbtIR9JNQF+y/PhE4GyyG8jcIqk/8A5wWDr8frIVU8eRrel0XFP1O3ibmRVUcMXAiDiixFN7NHBsACc2p34HbzMzoDnTANsCB28zswKvbWJmlkNt5EYL5XDwNjNbzCNvM7N8kW/GYGaWUx55m5nlj7+wNDPLm3zdjMHB28yswCNvM7MccvA2M8sbp03MzPLJI28zszxy8DYzyxdfpGNmllceeZuZ5Y8XpjIzyxuv521mlk+ebWJmlkdOm5iZ5YvwyNvMLH/kLyzNzHLJwdvMLG+8tomZWT5VMOctaTwwA1gILIiIPpK6AP8AegLjgcMi4uNlqT8/bzNmZi2upsytbLtFRO+I6JP2zwQei4hewGNpf5l7amZmkI28y9mW3f7AsPR4GHDAslbk4G1mBuUH7ix4d5U0qmgb0ECNATws6YWi59eNiMnp8XvAusvaXee8zcwWK3s8O6UoFVLK1yNikqR1gEckvV78ZESEpFiWXoJH3mZmS1QwbRIRk9LPD4A7ge2A9yV1z5pSd+CDZe2qg7eZ2WIqc2uiFmlVSZ0Lj4G9gFeAu4Fj02HHAncta0+dNjEzA7IrLCt2M4Z1gTuVjdLrgBsj4kFJzwO3SOoPvAMctqwNOHibmRVU6ArLiHgL+EoD5R8Be1SiDQdvMzPA63mbmeWVVxU0M8sZ4YWpzMzyySNvM7OcWe5L31uVg7eZ2WJOm5iZ5Y9H3mZmeVPRi3RanCKWeV2UFYakD8muhmrPugJTqt0Jq4gV4d/y8xHRrZIVSnqQ7HdXjikR0a+S7TeXg7cBIGlUGaukWQ7433LFkJ/svJmZLebgbWaWQw7eVjC02h2wivG/5QrAOW8zsxzyyNvMLIccvM3McsjBO4ckhaQLi/ZPkzSoBdr5eb39pyvdhi1N0kJJoyW9IulWSas08/yekr5btN9H0qWV76lVm4N3Ps0FDpJU7gUFy2qp4B0RX2vh9gxmR0TviNgCmAec0MzzewKLg3dEjIqIgRXsn7URDt75tIBsRsEp9Z+Q1E3S7ZKeT9tOReWPSBor6SpJ7xSCv6T/lfRCem5AKhsMrJxGgTekspnp582SvlnU5rWSDpFUK+mPqd0xko5v8d9E+/YksImkLunfaIykZyRtBSBp1/TvM1rSS+mGt4OBnVPZKZL6SrpXUo2k8ZLWLFQu6U1J65b6m7E2LiK85WwDZgKrA+OBNYDTgEHpuRuBr6fHGwKvpcd/As5Kj/sBAXRN+13Sz5XJ7nC9dqGd+u2mnwcCw9LjjsCEdO4A4JepvBMwCtio2r+vPG1Fv+M6sjuL/wi4DDg7le8OjE6P7wF2So9XS+f0Be4tqm/xPnAJcFx6vD3waGN/M97a9uaFqXIqIj6RdB0wEJhd9NQ3gM21ZHW01SWtBnydLOgS2V2sPy46Z6CkA9PjDYBewEeNNP8AcImkTmRvBCMiYrakvYCtJB2Sjlsj1fX2sr7OFdDKkkanx08CVwPPAgcDRMTjktaWtDowErgofTK6IyImqvFV8f4B/Br4G/CdtA8l/mYiYmYFX5dVmIN3vl0MvEj2n7GgBtghIuYUH1jqP7WkvmT/eXeMiFmShgMrNdZoRMxJx+0NHA7cXKgOOCkiHmruC7HFZkdE7+KCUv92ETFY0n3AvsBISXs3Ufe/yNIw3YADgN+l8gb/Zqxtc847xyJiKnAL0L+o+GHgpMKOpEIgGAkclsr2AtZK5WsAH6fAvSmwQ1Fd8yV1KNH8P4DjgJ2BB1PZQ8CPCudI+qKkVZfx5dkSTwJHwuI32ynpk9fGEfHviDgfeB7YFJgBdG6oksjyIncCF5GlRgqfrkr9zVgb5uCdfxey9DKWA4E+6cutV1kyW+EcYC9JrwCHAu+R/Ud/EKiT9BrZl13PFNU1FBhT+MKynoeBXcnypvNS2VXAq8CLqZ2/4k93lTAI2FbSGLJ/o2NT+U/TlMIxwHyydNYYYKGklyV95gttsjfdo1iSMoHSfzPWhvny+BVEyk8vjIgFknYErqj/8dzM8sOjohXHhsAtkmrI5g//sMr9MbPl4JG3mVkOOedtZpZDDt5mZjnk4G1mlkMO3tailneVvHp1XVu4ejOtz7J5I8f2ldTshbTS+h+fWfCrVHm9Y5p1RaKkQZJOa24fzcDB21peo6vkSVqmGU8R8YOIeLWRQ/oCXgXR2i0Hb2tNhVXy+kp6UtLdwKulViNU5k+S3pD0KLBOoSJJwyX1SY/7SXoxXZjymKSeZG8Sp6RR/86lVs5L64Q8rLTaItkl/o1SA6swFj03JJU/li5DR9LGkh5M5zyZrmQ1Wy6e522tIo2w92HJpfTbAFtExNspAE6PiK+mi4lGSnoY2Br4ErA5sC7Z1ZvX1Ku3G3AlsEuqq0tETJX0F7IV+i5Ix90IDImIpyRtSHYp/2bA2cBTEfEbZcvcFi81UMr3UxsrA89Luj1dar4qMCoiTpH061T3T8iuVD0hIt6UtD1wOdnqgGbLzMHbWlpDq+R9DXguIgqrDZZajXAX4KaIWAi8K+nxBurfgWxVw7dh8XovDSm12uIuwEHp3Pu09GqLpZRahXERSy47/ztwR2rja8CtRW13KqMNs0Y5eFtLK7VK3qfFRTSwGqGkfSvYj2attlhKM1dhjNTuNC9FYJXmnLe1BaVWIxwBHJ5y4t2B3Ro49xlgF0kbpXO7pPL6q+uVWjlvBOm2YZL2Yclqi6U0tgpjDVD49PBdsnTMJ8Dbkg5NbUjSV5pow6xJDt7WFpRajfBO4M303HVk61EvJSI+JLuDzx2SXmZJ2uIe4MDCF5Y0vtriLpLGkqVP/ttEXxtbhfFTYLv0GnYHfpPKjwT6p/6NBfYv43di1iivbWJmlkMeeZuZ5ZCDt5lZDjl4m5nlkIO3mVkOOXibmeWQg7eZWQ45eJuZ5dD/A8ibYYXsiph0AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KSw8BnHmyoXF"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}