Sentence_Classification_with_BERT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "name": "200361138_lab9.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.9"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "02c9c661ad664c36843c09854bc8b837": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_56fdead44e6a457dbf81fa8dd9ac77a1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e8e78e2b534c4ea589494944617ae463",
              "IPY_MODEL_f4c2bfa129a84e9793d88fceeb331653"
            ]
          }
        },
        "56fdead44e6a457dbf81fa8dd9ac77a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e8e78e2b534c4ea589494944617ae463": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6f510823f64f4d288b500beffad1d607",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_639db6f005fd4bb48377d1c6ffcdfb8f"
          }
        },
        "f4c2bfa129a84e9793d88fceeb331653": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ba2cf69b7a0943af9bebaae3a7afbfcb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:01&lt;00:00, 218kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8f8d422c7f4f46038e4f3ef8281c85f9"
          }
        },
        "6f510823f64f4d288b500beffad1d607": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "639db6f005fd4bb48377d1c6ffcdfb8f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ba2cf69b7a0943af9bebaae3a7afbfcb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8f8d422c7f4f46038e4f3ef8281c85f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3d54108e7a1b451ba5af58d652d67142": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6c4d28326c1b4559a3f6db37520126e2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ce7112f664b64feb8f35697dbdeb2f62",
              "IPY_MODEL_0bbcea48c5c44aa4a01db7d0f42365a6"
            ]
          }
        },
        "6c4d28326c1b4559a3f6db37520126e2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ce7112f664b64feb8f35697dbdeb2f62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_416048d6e9744bcf8317fba2ec55405a",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4752364e35bc42d69630603de6c6dded"
          }
        },
        "0bbcea48c5c44aa4a01db7d0f42365a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_af5e9acaac7e49fd9121ca27a15d5e0d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:00&lt;00:00, 58.3B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6d766228db704469a064b77662a2f5f6"
          }
        },
        "416048d6e9744bcf8317fba2ec55405a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4752364e35bc42d69630603de6c6dded": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "af5e9acaac7e49fd9121ca27a15d5e0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6d766228db704469a064b77662a2f5f6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1c87dd3484614496a8d3a3ab96e09248": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_25e74bb79dc24f148b8c0c00713f33ad",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ededdd252ad94649925ec3967f217b6d",
              "IPY_MODEL_cc365e912f314ea289dd0e83a9573a4c"
            ]
          }
        },
        "25e74bb79dc24f148b8c0c00713f33ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ededdd252ad94649925ec3967f217b6d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d46b6d0a9e9f456ca43c3da097c6685a",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6d336539dba3424781fbebabbab76d5a"
          }
        },
        "cc365e912f314ea289dd0e83a9573a4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_68eecec4b77c49a4bad4959202cc2320",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:00&lt;00:00, 2.23MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8117e9af9949497db42822238bd41a07"
          }
        },
        "d46b6d0a9e9f456ca43c3da097c6685a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6d336539dba3424781fbebabbab76d5a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "68eecec4b77c49a4bad4959202cc2320": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8117e9af9949497db42822238bd41a07": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e753b302bcbe47c59128c768c12c7fc4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ffd10ea8f6894162b95ad4ae75371576",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_519a9f3cca084d748715c829f8a09cbb",
              "IPY_MODEL_27790dd6deb14b06b154b3c857c5eefe"
            ]
          }
        },
        "ffd10ea8f6894162b95ad4ae75371576": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "519a9f3cca084d748715c829f8a09cbb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_52c22ab043de4c58b2f87d2f9d2979c6",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 363423424,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 363423424,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7faa7fd91a624fe59352a854b7239209"
          }
        },
        "27790dd6deb14b06b154b3c857c5eefe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e792739a24184e92ac949b5a144ac414",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 363M/363M [00:08&lt;00:00, 44.8MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_23c67f188f7e42e98cd3f70b7d2fde28"
          }
        },
        "52c22ab043de4c58b2f87d2f9d2979c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7faa7fd91a624fe59352a854b7239209": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e792739a24184e92ac949b5a144ac414": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "23c67f188f7e42e98cd3f70b7d2fde28": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pWgujXmGqzIC"
      },
      "source": [
        "# Sentence Classification with BERT"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pr5PWYKGPi6R"
      },
      "source": [
        "In this lab we turn a pre-trained BERT model into a trainable Keras layer and apply it to the semeval 2017 task 4 subtask B that we tackled in lab 5. BERT (Bidirectional Embedding Representations from Transformers) is a new model for pre-training language representations that obtains state-of-the-art results on many NLP tasks. We demonstrate how to integrate BERT as a custom Keras layer to simplify model prototyping using huggingface. In this lab, you will learn: \n",
        "\n",
        "1) How to use the huggingface package.\n",
        "\n",
        "2) How to integrate BERT in our previous model. \n",
        "\n",
        "3) How to use the TPU from Colab. (Note: Running the BERT on the CPU would be very slow. Thus we recommend you to do this lab on Colab based on TPU provided by Google.)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4spJ5PRhGJ1l"
      },
      "source": [
        "import keras\n",
        "import numpy as np\n",
        "from keras.layers import Lambda, GlobalAveragePooling1D, Dense, Embedding\n",
        "from keras import backend as K\n",
        "from keras.models import Sequential\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from keras.layers import LSTM, RNN, Dropout, Input, LeakyReLU, Bidirectional,Conv1D, GlobalMaxPooling1D\n",
        "from keras.layers.core import Dense\n",
        "from keras.models import Model\n",
        "from keras.utils import plot_model\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2oimYsLssuSs"
      },
      "source": [
        "Before we start, we should install the huggingface transformer package. You can find the doc from its [website](https://huggingface.co/transformers/index.html)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AElpzsKFiZSo",
        "outputId": "81b8e238-4e4a-493f-8c32-9f032df79a69"
      },
      "source": [
        "pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d8/b2/57495b5309f09fa501866e225c84532d1fd89536ea62406b2181933fb418/transformers-4.5.1-py3-none-any.whl (2.1MB)\n",
            "\u001b[K     |████████████████████████████████| 2.1MB 4.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.10.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/08/cd/342e584ee544d044fb573ae697404ce22ede086c9e87ce5960772084cad0/sacremoses-0.0.44.tar.gz (862kB)\n",
            "\u001b[K     |████████████████████████████████| 870kB 21.3MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 31.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.44-cp37-none-any.whl size=886084 sha256=e95791b31a5630365ce8ab1ebe4ac8430679eb7250367c31a875fa67b8c2fe1f\n",
            "  Stored in directory: /root/.cache/pip/wheels/3e/fb/c0/13ab4d63d537658f448366744654323077c4d90069b6512f3c\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.44 tokenizers-0.10.2 transformers-4.5.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ebbamBD0xu5z"
      },
      "source": [
        "## Preprocessing and Tokenization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RbAMXQwqyVT8"
      },
      "source": [
        "In this lab we will use DistilBERT instead of BERT: DistilBERT is a small, fast, cheap and light Transformer model trained by distilling BERT base. It has 40% less parameters than bert-base-uncased, and runs 60% faster, while preserving over 95% of BERT’s performance as measured on the GLUE language understanding benchmark.\n",
        "\n",
        "It is easy to switch between DistilBERT and BERT using the huggingface transformer package. This huggingface package provides many pre-trained and pre-built models that are easy to use via a few lines of code."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V-cUTxt02dQb"
      },
      "source": [
        "Before using DistilBERT or BERT, we need a tokenizer. Generally speaking, every BERT related model has its own tokenizer, trained for that model (see this week's lecture video on sub-word tokenization). \n",
        "We can get the DistilBERT tokenizer from **DistilBertTokenizer.from_pretrained** function."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hKj6Y_TydjeS",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 163,
          "referenced_widgets": [
            "02c9c661ad664c36843c09854bc8b837",
            "56fdead44e6a457dbf81fa8dd9ac77a1",
            "e8e78e2b534c4ea589494944617ae463",
            "f4c2bfa129a84e9793d88fceeb331653",
            "6f510823f64f4d288b500beffad1d607",
            "639db6f005fd4bb48377d1c6ffcdfb8f",
            "ba2cf69b7a0943af9bebaae3a7afbfcb",
            "8f8d422c7f4f46038e4f3ef8281c85f9",
            "3d54108e7a1b451ba5af58d652d67142",
            "6c4d28326c1b4559a3f6db37520126e2",
            "ce7112f664b64feb8f35697dbdeb2f62",
            "0bbcea48c5c44aa4a01db7d0f42365a6",
            "416048d6e9744bcf8317fba2ec55405a",
            "4752364e35bc42d69630603de6c6dded",
            "af5e9acaac7e49fd9121ca27a15d5e0d",
            "6d766228db704469a064b77662a2f5f6",
            "1c87dd3484614496a8d3a3ab96e09248",
            "25e74bb79dc24f148b8c0c00713f33ad",
            "ededdd252ad94649925ec3967f217b6d",
            "cc365e912f314ea289dd0e83a9573a4c",
            "d46b6d0a9e9f456ca43c3da097c6685a",
            "6d336539dba3424781fbebabbab76d5a",
            "68eecec4b77c49a4bad4959202cc2320",
            "8117e9af9949497db42822238bd41a07"
          ]
        },
        "outputId": "6f2250f8-b173-4eaf-d467-03a071aca58a"
      },
      "source": [
        "from transformers import DistilBertTokenizer, RobertaTokenizer \n",
        "import tqdm\n",
        "distil_bert = 'distilbert-base-uncased' # Pick any desired pre-trained model\n",
        "\n",
        "# Defining DistilBERT tokonizer\n",
        "tokenizer = DistilBertTokenizer.from_pretrained(distil_bert, do_lower_case=True, add_special_tokens=True,\n",
        "                                                max_length=128, pad_to_max_length=True)\n",
        "\n",
        "def tokenize(sentences, tokenizer, pad_length=128, pad_to_max_length=True ):\n",
        "    if type(sentences) == str:\n",
        "        inputs = tokenizer.encode_plus(sentences, add_special_tokens=True, max_length=pad_length, pad_to_max_length=pad_to_max_length, \n",
        "                                             return_attention_mask=True, return_token_type_ids=True)\n",
        "        return np.asarray(inputs['input_ids'], dtype='int32'), np.asarray(inputs['attention_mask'], dtype='int32'), np.asarray(inputs['token_type_ids'], dtype='int32')\n",
        "    input_ids, input_masks, input_segments = [],[],[]\n",
        "    for sentence in sentences:\n",
        "        inputs = tokenizer.encode_plus(sentence, add_special_tokens=True, max_length=pad_length, pad_to_max_length=pad_to_max_length, \n",
        "                                             return_attention_mask=True, return_token_type_ids=True)\n",
        "        input_ids.append(inputs['input_ids'])\n",
        "        input_masks.append(inputs['attention_mask'])\n",
        "        input_segments.append(inputs['token_type_ids'])        \n",
        "        \n",
        "    return np.asarray(input_ids, dtype='int32'), np.asarray(input_masks, dtype='int32'), np.asarray(input_segments, dtype='int32')\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "02c9c661ad664c36843c09854bc8b837",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3d54108e7a1b451ba5af58d652d67142",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1c87dd3484614496a8d3a3ab96e09248",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sqMo6CPS3qJZ"
      },
      "source": [
        "Then we can use the tokenizer to tokenize the sentence. When working with word2vec and GloVe, we tokenized the sentence into words ourselves and then converted the tokens to GloVe word indices. But in BERT, we must use the BERT tokenizer: the tokens for BERT are different, and include whole words and sub-word tokens (see lecture video on sub-word tokenisation).\n",
        "\n",
        "For example, for the sentence: **This is a pretrained model.** our previous word-based tokenizer will generate the following tokens:\n",
        "\n",
        "**\"this\", \"is\", \"a\", \"pretrained\", \"model\", \".\"**\n",
        "\n",
        "Then you will find out that the word token \"pretrained\" is not in the GloVe word dictionary. Thus we can not assign a proper word vector for \"pretrained\".\n",
        "\n",
        "In BERT, the BERT tokenizer will separate the word \"pretrained\" into three sub-word tokens:\n",
        "\n",
        "**'pre', '##train', '##ed'**\n",
        "\n",
        "This way, BERT can use these three token vectors to represent the word \"pretrained\". Without the BERT tokenizer, it is hard to separate these unknown words properly."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iRoKe2DKyi41",
        "outputId": "2c3aefe9-9205-42a0-907e-9b580ae4759c"
      },
      "source": [
        "inputs = tokenizer.tokenize(\"The capital of France is [MASK].\")\n",
        "print(inputs,'\\n')\n",
        "\n",
        "inputs = tokenizer.tokenize(\"This is a pretrained model.\")\n",
        "print(inputs,'\\n')\n",
        "\n",
        "ids,masks,segments = tokenize(\"The capital of France is [MASK].\", tokenizer)\n",
        "print(ids)\n",
        "print(masks,\"\\n\")\n",
        "\n",
        "ids,masks,segments = tokenize(\"The capital of France is [MASK].\", tokenizer, pad_to_max_length=False)\n",
        "print(ids)\n",
        "print(masks)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "['the', 'capital', 'of', 'france', 'is', '[MASK]', '.'] \n",
            "\n",
            "['this', 'is', 'a', 'pre', '##train', '##ed', 'model', '.'] \n",
            "\n",
            "[ 101 1996 3007 1997 2605 2003  103 1012  102    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0]\n",
            "[1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] \n",
            "\n",
            "[ 101 1996 3007 1997 2605 2003  103 1012  102]\n",
            "[1 1 1 1 1 1 1 1 1]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2079: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VdWp7ajV0B-O"
      },
      "source": [
        "An example of a tokenize function."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J6OuZAA8sbdg"
      },
      "source": [
        "## Loading Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cqvPQvgvPv1W"
      },
      "source": [
        "### Downloading and preprocessing data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EundMtGPpCdf"
      },
      "source": [
        "Similar to lab 5, we need to download and preprocess the data first. The data download code is consistent with lab 5."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NyuSzkafqNca",
        "outputId": "5478d04e-2b0c-48e4-d205-3b37b5d08587"
      },
      "source": [
        "import requests\n",
        "def downloadfile(url):\n",
        "  rq = requests.get(url)\n",
        "  open(url.split('/')[-1], 'wb').write(rq.content)\n",
        "\n",
        "downloadfile('https://raw.githubusercontent.com/cbaziotis/datastories-semeval2017-task4/master/dataset/Subtask_BD/downloaded/twitter-2016devtest-BD.tsv')\n",
        "downloadfile('https://raw.githubusercontent.com/cbaziotis/datastories-semeval2017-task4/master/dataset/Subtask_BD/downloaded/twitter-2016dev-BD.tsv')\n",
        "downloadfile('https://raw.githubusercontent.com/cbaziotis/datastories-semeval2017-task4/master/dataset/Subtask_BD/downloaded/twitter-2016test-BD.tsv')\n",
        "downloadfile('https://raw.githubusercontent.com/cbaziotis/datastories-semeval2017-task4/master/dataset/Subtask_BD/downloaded/twitter-2016train-BD.tsv')\n",
        "downloadfile('https://raw.githubusercontent.com/cbaziotis/datastories-semeval2017-task4/master/dataset/Subtask_BD/downloaded/twitter-2015test-BD.tsv')\n",
        "downloadfile('https://raw.githubusercontent.com/cbaziotis/datastories-semeval2017-task4/master/dataset/Subtask_BD/downloaded/twitter-2015train-BD.tsv')\n",
        "\n",
        "downloadfile('https://raw.githubusercontent.com/cbaziotis/datastories-semeval2017-task4/master/dataset/Subtask_BD/gold/SemEval2017-task4-test.subtask-BD.english.txt')\n",
        "\n",
        "with open('twitter-2016dev-BD.tsv', 'r') as f:\n",
        "  dev_original = [l.strip().split('\\t') for l in f.readlines()]\n",
        "with open('SemEval2017-task4-test.subtask-BD.english.txt', 'r') as f:\n",
        "  test_original = [l.strip().split('\\t') for l in f.readlines()]\n",
        "train_original = []\n",
        "with open('twitter-2016train-BD.tsv', 'r') as f:\n",
        "  train_original = [l.strip().split('\\t') for l in f.readlines()]\n",
        "with open('twitter-2016test-BD.tsv', 'r') as f:\n",
        "  train_original.extend([l.strip().split('\\t') for l in f.readlines()])\n",
        "with open('twitter-2016devtest-BD.tsv', 'r') as f:\n",
        "  train_original.extend([l.strip().split('\\t') for l in f.readlines()])\n",
        "with open('twitter-2015test-BD.tsv', 'r') as f:\n",
        "  train_original.extend([l.strip().split('\\t') for l in f.readlines() if l.strip().split('\\t')[2] in ['negative','positive']])\n",
        "with open('twitter-2015train-BD.tsv', 'r') as f:\n",
        "  train_original.extend([l.strip().split('\\t') for l in f.readlines() if l.strip().split('\\t')[2] in ['negative','positive']])\n",
        "\n",
        "print(\"Training entries: {}\".format(len(train_original)))\n",
        "print(\"Development entries: {}\".format(len(dev_original)))\n",
        "print(\"Testing entries: {}\".format(len(test_original)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training entries: 17639\n",
            "Development entries: 1325\n",
            "Testing entries: 6185\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6U4iCV9-rmay"
      },
      "source": [
        "We now can start playing around with the data, let’s first see some examples:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h-gjWRAuqg5s",
        "outputId": "0df8eaea-5975-4c14-dfe0-78160223c4ed"
      },
      "source": [
        "print(\"ID \\t TOPIC \\t LABLE \\t TWEET_TEXT\")\n",
        "print(train_original[0])\n",
        "print(train_original[1])\n",
        "print(train_original[2])\n",
        "print(train_original[3])\n",
        "print(train_original[4])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ID \t TOPIC \t LABLE \t TWEET_TEXT\n",
            "['628949369883000832', '@microsoft', 'negative', \"dear @Microsoft the newOoffice for Mac is great and all, but no Lync update? C'mon.\"]\n",
            "['628976607420645377', '@microsoft', 'negative', \"@Microsoft how about you make a system that doesn't eat my friggin discs. This is the 2nd time this has happened and I am so sick of it!\"]\n",
            "['629023169169518592', '@microsoft', 'negative', \"I may be ignorant on this issue but... should we celebrate @Microsoft's parental leave changes? Doesn't the gender divide suggest... (1/2)\"]\n",
            "['629179223232479232', '@microsoft', 'negative', 'Thanks to @microsoft, I just may be switching over to @apple.']\n",
            "['629226490152914944', '@microsoft', 'positive', 'Microsoft, I may not prefer your gaming branch of business. But, you do make a damn fine operating system. #Windows10 @Microsoft']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tvuu4KhStqei"
      },
      "source": [
        "According to the BERT tokenize function above, we can convert the tweet text and topic words to integers:\n",
        "\n",
        "(Note: as explained above, the BERT tokenize function is different from lab 5.)\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gMCH1OoDrSNR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1e722307-f41c-4016-8a32-a4d3bbdcf4ff"
      },
      "source": [
        "# Please write your code to generate the following data\n",
        "x_train_tweet_int = []\n",
        "x_train_tweet_masks = []\n",
        "x_train_topic_int = []\n",
        "x_train_topic_masks = []\n",
        "\n",
        "x_dev_tweet_int = []\n",
        "x_dev_tweet_masks = []\n",
        "x_dev_topic_int = []\n",
        "x_dev_topic_masks = []\n",
        "\n",
        "x_test_tweet_int = []\n",
        "x_test_tweet_masks = []\n",
        "x_test_topic_int = []\n",
        "x_test_topic_masks = []\n",
        "\n",
        "\n",
        "for record in train_original:\n",
        "  ids,masks,segments = tokenize(record[3], tokenizer, pad_to_max_length=False)\n",
        "  x_train_tweet_int.append(ids)\n",
        "  x_train_tweet_masks.append(masks)\n",
        "  ids,masks,segments = tokenize(record[1], tokenizer, pad_to_max_length=False)\n",
        "  x_train_topic_int.append(ids)\n",
        "  x_train_topic_masks.append(masks)\n",
        "\n",
        "\n",
        "x_dev_tweet_int = []\n",
        "x_dev_tweet_masks = []\n",
        "x_dev_topic_int = []\n",
        "x_dev_topic_masks = []\n",
        "for record in dev_original:\n",
        "  ids,masks,segments = tokenize(record[3], tokenizer, pad_to_max_length=False)\n",
        "  x_dev_tweet_int.append(ids)\n",
        "  x_dev_tweet_masks.append(masks)\n",
        "  ids,masks,segments = tokenize(record[1], tokenizer, pad_to_max_length=False)\n",
        "  x_dev_topic_int.append(ids)\n",
        "  x_dev_topic_masks.append(masks)\n",
        "\n",
        "x_test_tweet_int = []\n",
        "x_test_tweet_masks = []\n",
        "x_test_topic_int = []\n",
        "x_test_topic_masks = []\n",
        "for record in test_original:\n",
        "  ids,masks,segments = tokenize(record[3], tokenizer, pad_to_max_length=False)\n",
        "  x_test_tweet_int.append(ids)\n",
        "  x_test_tweet_masks.append(masks)\n",
        "  ids,masks,segments = tokenize(record[1], tokenizer, pad_to_max_length=False)\n",
        "  x_test_topic_int.append(ids)\n",
        "  x_test_topic_masks.append(masks)\n",
        "\n",
        "# If use the previous tokenize function, you can get a print result like:\n",
        "assert len(x_train_topic_int) == len(train_original)\n",
        "assert len(x_train_topic_masks) == len(x_train_topic_int)\n",
        "assert len(x_test_topic_int) == len(test_original)\n",
        "assert len(x_test_topic_masks) == len(x_test_topic_int)\n",
        "print(\"x_dev_topic_int[0]:\")\n",
        "print(x_dev_topic_int[0])\n",
        "print(\"x_dev_topic_masks[0]:\")\n",
        "print(x_dev_topic_masks[0])\n",
        "print(\"x_dev_tweet_int[0]:\")\n",
        "print(x_dev_tweet_int[0])\n",
        "print(\"x_dev_tweet_masks[0]:\")\n",
        "print(x_dev_tweet_masks[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "x_dev_topic_int[0]:\n",
            "[ 101 2745 4027  102]\n",
            "x_dev_topic_masks[0]:\n",
            "[1 1 1 1]\n",
            "x_dev_tweet_int[0]:\n",
            "[  101  6108  1062  9794 16021 23091  2007 16839  9080 12863  7050  2000\n",
            "  2745  4027  1024  6108  1062  4593  2587 16021 23091  2006  5095  1998\n",
            "  1012  1012  8299  1024  1013  1013  1056  1012  2522  1013  1053  3501\n",
            "  2683  2072  2549  8586  2615 18037   102]\n",
            "x_dev_tweet_masks[0]:\n",
            "[1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            " 1 1 1 1 1 1]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8j7uVWBRGsny"
      },
      "source": [
        "We are converting the tweet text and topic words to integers using the bert tokenize function. After converting them to integers we are appending them to the list of trian_tweet_int and train_tweet_topic for the training data and similary we will append for development data and test data. Also for topic train_topic_int and train_topic_mask for training, development, and test data. Adter that we check the Assertiom. If they are of the same length."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5IreFXgruZot"
      },
      "source": [
        "We use 1 to represent \"positive\" and 0 for \"negative\" and generate the \"y\" data similar to previous labs."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "abIb7Fe5u3GQ",
        "outputId": "c23d4a77-f5ef-4a4e-ad34-aec23a499a6b"
      },
      "source": [
        "def label2int(dataset):\n",
        "  y = []\n",
        "  for example in dataset:\n",
        "    if example[2].lower() == \"negative\":\n",
        "      y.append(0)\n",
        "    else:\n",
        "      y.append(1)\n",
        "  return y\n",
        "  \n",
        "y_train = label2int(train_original)\n",
        "y_dev = label2int(dev_original)\n",
        "y_test = label2int(test_original)\n",
        "y_train = np.array(y_train)\n",
        "y_dev = np.array(y_dev)\n",
        "y_test = np.array(y_test)\n",
        "\n",
        "print(y_train[0])\n",
        "print(y_train[1])\n",
        "print(y_train[2])\n",
        "print(y_train[3])\n",
        "print(y_train[4])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0\n",
            "0\n",
            "0\n",
            "0\n",
            "1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "txxIQPEdFSgY"
      },
      "source": [
        "We also generate these target labels using one hot encoding (This will be used in model 1)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ohDm-E7k2w6c",
        "outputId": "3bc5fd51-d360-4882-b490-f418addf7091"
      },
      "source": [
        "def int2onehot(dataset):\n",
        "  y = []\n",
        "  for example in dataset:\n",
        "    if example:\n",
        "      y.append(np.array([0,1]))\n",
        "    else:\n",
        "      y.append(np.array([1,0]))\n",
        "  return np.array(y)\n",
        "y_train_onehot = int2onehot(y_train)\n",
        "y_dev_onehot = int2onehot(y_dev)\n",
        "y_test_onehot = int2onehot(y_test)\n",
        "\n",
        "print(y_train_onehot[0])\n",
        "print(y_train_onehot[1])\n",
        "print(y_train_onehot[2])\n",
        "print(y_train_onehot[3])\n",
        "print(y_train_onehot[4])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1 0]\n",
            "[1 0]\n",
            "[1 0]\n",
            "[1 0]\n",
            "[0 1]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9TnnSuspvC5b"
      },
      "source": [
        "Now we have almost done the data preprocessing. Unlike the previous lab1-lab4, there are two distinct parts to the input x (tweet and topic - see lab 5) which we must input to the model. The easiest way is to input the tweet and topic as paired sentences into the model, concatenating them and separating them by the special BERT sentence-separator token [SEP] (available as tokenizer.sep_token). Then we can apply BERT directly."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nKOiVVXQu-_I",
        "outputId": "ad68e1d3-cf83-475b-e6aa-396c33981427"
      },
      "source": [
        "# Please write your code to combine the tweet and topic into the following varibles\n",
        "# x_train_int\n",
        "# x_train_masks\n",
        "# x_dev_int\n",
        "# x_dev_masks\n",
        "# x_test_int\n",
        "# x_test_masks\n",
        "\n",
        "# Tips: \n",
        "# 1) We can use the special token <SEP> to concatenate the tweets and topics.\n",
        "# 2) After combine them, make sure they are paded.\n",
        "\n",
        "x_train_data = []\n",
        "for record in train_original:\n",
        "  x_train_data.append(record[3] + ' ' + tokenizer.sep_token + ' '+ record[1])\n",
        "\n",
        "x_dev_data = []\n",
        "for record in dev_original:\n",
        "  x_dev_data.append(record[3] + ' ' + tokenizer.sep_token + ' '+ record[1])\n",
        "\n",
        "x_test_data = []\n",
        "for record in test_original:\n",
        "  x_test_data.append(record[3] + ' ' + tokenizer.sep_token + ' '+ record[1])\n",
        "\n",
        "x_train_int = []\n",
        "x_train_masks = []\n",
        "for record in x_train_data:\n",
        "  ids,masks,segments = tokenize(record, tokenizer, pad_to_max_length=False)\n",
        "  x_train_int.append(ids)\n",
        "  x_train_masks.append(masks)\n",
        "\n",
        "x_dev_int = []\n",
        "x_dev_masks = []\n",
        "for record in x_dev_data:\n",
        "  ids,masks,segments = tokenize(record, tokenizer, pad_to_max_length=False)\n",
        "  x_dev_int.append(ids)\n",
        "  x_dev_masks.append(masks)\n",
        "\n",
        "x_test_int = []\n",
        "x_test_masks = []\n",
        "for record in x_test_data:\n",
        "  ids,masks,segments = tokenize(record, tokenizer, pad_to_max_length=False)\n",
        "  x_test_int.append(ids)\n",
        "  x_test_masks.append(masks)\n",
        "\n",
        "\n",
        "\n",
        "x_train_int = np.array(keras.preprocessing.sequence.pad_sequences(x_train_int, padding='post', maxlen=128, value=0))\n",
        "x_dev_int = np.array(keras.preprocessing.sequence.pad_sequences(x_dev_int, padding='post', maxlen=128, value=0))\n",
        "x_test_int = np.array(keras.preprocessing.sequence.pad_sequences(x_test_int, padding='post', maxlen=128, value=0))\n",
        "x_train_masks = np.array(keras.preprocessing.sequence.pad_sequences(x_train_masks, padding='post', maxlen=128, value=0))\n",
        "x_dev_masks = np.array(keras.preprocessing.sequence.pad_sequences(x_dev_masks, padding='post', maxlen=128, value=0))\n",
        "x_test_masks = np.array(keras.preprocessing.sequence.pad_sequences(x_test_masks, padding='post', maxlen=128, value=0))\n",
        "\n",
        "\n",
        "# your code goes here\n",
        "\n",
        "\n",
        "\n",
        "# Don't forget the to use np.array function to wrap the ouput of pad_sequences function\n",
        "x_train_int_np = np.array(x_train_int)\n",
        "x_train_masks_np = np.array(x_train_masks)\n",
        "x_dev_int_np = np.array(x_dev_int)\n",
        "x_dev_masks_np = np.array(x_dev_masks)\n",
        "x_test_int_np = np.array(x_test_int)\n",
        "x_test_masks_np = np.array(x_test_masks)\n",
        "\n",
        "\n",
        "print(x_dev_int[0])\n",
        "print(x_dev_masks[0],'\\n')\n",
        "print(x_dev_int_np[0])\n",
        "print(x_dev_masks_np[0])\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[  101  6108  1062  9794 16021 23091  2007 16839  9080 12863  7050  2000\n",
            "  2745  4027  1024  6108  1062  4593  2587 16021 23091  2006  5095  1998\n",
            "  1012  1012  8299  1024  1013  1013  1056  1012  2522  1013  1053  3501\n",
            "  2683  2072  2549  8586  2615 18037   102  2745  4027   102     0     0\n",
            "     0     0     0     0     0     0     0     0     0     0     0     0\n",
            "     0     0     0     0     0     0     0     0     0     0     0     0\n",
            "     0     0     0     0     0     0     0     0     0     0     0     0\n",
            "     0     0     0     0     0     0     0     0     0     0     0     0\n",
            "     0     0     0     0     0     0     0     0     0     0     0     0\n",
            "     0     0     0     0     0     0     0     0     0     0     0     0\n",
            "     0     0     0     0     0     0     0     0]\n",
            "[1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            " 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] \n",
            "\n",
            "[  101  6108  1062  9794 16021 23091  2007 16839  9080 12863  7050  2000\n",
            "  2745  4027  1024  6108  1062  4593  2587 16021 23091  2006  5095  1998\n",
            "  1012  1012  8299  1024  1013  1013  1056  1012  2522  1013  1053  3501\n",
            "  2683  2072  2549  8586  2615 18037   102  2745  4027   102     0     0\n",
            "     0     0     0     0     0     0     0     0     0     0     0     0\n",
            "     0     0     0     0     0     0     0     0     0     0     0     0\n",
            "     0     0     0     0     0     0     0     0     0     0     0     0\n",
            "     0     0     0     0     0     0     0     0     0     0     0     0\n",
            "     0     0     0     0     0     0     0     0     0     0     0     0\n",
            "     0     0     0     0     0     0     0     0     0     0     0     0\n",
            "     0     0     0     0     0     0     0     0]\n",
            "[1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
            " 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FBOYCgxjHibg"
      },
      "source": [
        "Here we are concatenating tweet and topic as a pair of sentence adn we are separating it with the tokenizer.sep_token, and adding padding to sequences and then wrapping the output of the pad_sequence function using numpy."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tMovowEarm81"
      },
      "source": [
        "In your report, show a few examples of your input, with text and topic included but separated by [SEP]."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UqvUGIwwGJqu"
      },
      "source": [
        "## Model 1: Prebuilt Sequence Classification\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hSxC41ln07im"
      },
      "source": [
        "The huggingface transformer package provides many prebuilt models. First, let us try a basic text classification model based on DistilBERT. This first method is the standard way BERT models are used for text classification: we use the embedding of the special [CLS] token at the beginning of the sequence, and put it through a simple classifier layer to make the decision. The TFDistilBertForSequenceClassification class takes care of that for us.\n",
        "\n",
        "The models with BERT are much bigger than our previous models. To run it faster, we can use TPU here. The detailed guideline about using TPU can be found from https://www.tensorflow.org/guide/tpu."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "e753b302bcbe47c59128c768c12c7fc4",
            "ffd10ea8f6894162b95ad4ae75371576",
            "519a9f3cca084d748715c829f8a09cbb",
            "27790dd6deb14b06b154b3c857c5eefe",
            "52c22ab043de4c58b2f87d2f9d2979c6",
            "7faa7fd91a624fe59352a854b7239209",
            "e792739a24184e92ac949b5a144ac414",
            "23c67f188f7e42e98cd3f70b7d2fde28"
          ]
        },
        "id": "1gXFbb2cxBlw",
        "outputId": "e5e939f2-1c8c-4183-d986-d00576582134"
      },
      "source": [
        "from transformers import TFDistilBertForSequenceClassification, DistilBertConfig\n",
        "import tensorflow as tf\n",
        "\n",
        "distil_bert = 'distilbert-base-uncased'\n",
        "\n",
        "config = DistilBertConfig(num_labels=2)\n",
        "config.output_hidden_states = False\n",
        "\n",
        "def create_TFDistilBertForSequenceClassification():\n",
        "  transformer_model = TFDistilBertForSequenceClassification.from_pretrained(distil_bert, config = config)\n",
        "  input_ids = tf.keras.layers.Input(shape=(128,), name='input_token', dtype='int32')\n",
        "  input_masks_ids = tf.keras.layers.Input(shape=(128,), name='masked_token', dtype='int32')\n",
        "  X = transformer_model(input_ids, input_masks_ids)\n",
        "  return tf.keras.Model(inputs=[input_ids, input_masks_ids], outputs = X)\n",
        "\n",
        "use_tpu = True\n",
        "if use_tpu:\n",
        "  # Create distribution strategy\n",
        "  tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
        "  tf.config.experimental_connect_to_cluster(tpu)\n",
        "  tf.tpu.experimental.initialize_tpu_system(tpu)\n",
        "  strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
        "\n",
        "  # Create model on TPU:\n",
        "  with strategy.scope():\n",
        "    model = create_TFDistilBertForSequenceClassification()\n",
        "    optimizer = keras.optimizers.Adam(lr=5e-5)\n",
        "    model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])\n",
        "else:\n",
        "  model = create_TFDistilBertForSequenceClassification()\n",
        "  model.compile(optimizer='adam',\n",
        "              loss='binary_crossentropy',\n",
        "              metrics=['accuracy'])\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:absl:Entering into master device scope: /job:worker/replica:0/task:0/device:CPU:0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.113.134.202:8470\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.113.134.202:8470\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n",
            "WARNING:absl:`tf.distribute.experimental.TPUStrategy` is deprecated, please use  the non experimental symbol `tf.distribute.TPUStrategy` instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e753b302bcbe47c59128c768c12c7fc4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=363423424.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertForSequenceClassification: ['vocab_layer_norm', 'vocab_projector', 'vocab_transform', 'activation_13']\n",
            "- This IS expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some layers of TFDistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier', 'dropout_19', 'pre_classifier']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "WARNING:tensorflow:AutoGraph could not transform <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7f323ee47d70>> and will run it as-is.\n",
            "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
            "Cause: module, class, method, function, traceback, frame, or code object was expected, got cython_function_or_method\n",
            "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:AutoGraph could not transform <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7f323ee47d70>> and will run it as-is.\n",
            "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
            "Cause: module, class, method, function, traceback, frame, or code object was expected, got cython_function_or_method\n",
            "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING: AutoGraph could not transform <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7f323ee47d70>> and will run it as-is.\n",
            "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
            "Cause: module, class, method, function, traceback, frame, or code object was expected, got cython_function_or_method\n",
            "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
            "WARNING:tensorflow:AutoGraph could not transform <function wrap at 0x7f325a6f1c20> and will run it as-is.\n",
            "Cause: while/else statement not yet supported\n",
            "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "WARNING:tensorflow:AutoGraph could not transform <function wrap at 0x7f325a6f1c20> and will run it as-is.\n",
            "Cause: while/else statement not yet supported\n",
            "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
            "WARNING: AutoGraph could not transform <function wrap at 0x7f325a6f1c20> and will run it as-is.\n",
            "Cause: while/else statement not yet supported\n",
            "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2CPFj0CMx9mw",
        "outputId": "887a7ec9-a324-4a5e-c5e8-2be2f53207e8"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_token (InputLayer)        [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "masked_token (InputLayer)       [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "tf_distil_bert_for_sequence_cla TFSequenceClassifier 66955010    input_token[0][0]                \n",
            "                                                                 masked_token[0][0]               \n",
            "==================================================================================================\n",
            "Total params: 66,955,010\n",
            "Trainable params: 66,955,010\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zQQH5lE_33Vn",
        "outputId": "0f837334-bde5-4fae-bf61-c992dad46c88"
      },
      "source": [
        "history = model.fit([x_train_int_np,x_train_masks_np],\n",
        "                    y_train_onehot,\n",
        "                    epochs=30,\n",
        "                    batch_size=512,\n",
        "                    validation_data=([x_dev_int_np,x_dev_masks_np], y_dev_onehot),\n",
        "                    verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/30\n",
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "35/35 [==============================] - ETA: 0s - loss: 0.9401 - accuracy: 0.7304WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r35/35 [==============================] - 92s 1s/step - loss: 0.9303 - accuracy: 0.7320 - val_loss: 0.3989 - val_accuracy: 0.8355\n",
            "Epoch 2/30\n",
            "35/35 [==============================] - 7s 202ms/step - loss: 0.3772 - accuracy: 0.8679 - val_loss: 0.3912 - val_accuracy: 0.8445\n",
            "Epoch 3/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.4008 - accuracy: 0.8416 - val_loss: 0.4639 - val_accuracy: 0.7442\n",
            "Epoch 4/30\n",
            "35/35 [==============================] - 7s 202ms/step - loss: 0.3499 - accuracy: 0.8109 - val_loss: 0.3852 - val_accuracy: 0.8377\n",
            "Epoch 5/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.2770 - accuracy: 0.8997 - val_loss: 0.3752 - val_accuracy: 0.8491\n",
            "Epoch 6/30\n",
            "35/35 [==============================] - 7s 202ms/step - loss: 0.2209 - accuracy: 0.9187 - val_loss: 0.4071 - val_accuracy: 0.8355\n",
            "Epoch 7/30\n",
            "35/35 [==============================] - 7s 204ms/step - loss: 0.2259 - accuracy: 0.9214 - val_loss: 0.5884 - val_accuracy: 0.8408\n",
            "Epoch 8/30\n",
            "35/35 [==============================] - 7s 204ms/step - loss: 0.2078 - accuracy: 0.9268 - val_loss: 0.3857 - val_accuracy: 0.8528\n",
            "Epoch 9/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.1740 - accuracy: 0.9431 - val_loss: 0.5501 - val_accuracy: 0.8430\n",
            "Epoch 10/30\n",
            "35/35 [==============================] - 7s 202ms/step - loss: 0.1323 - accuracy: 0.9560 - val_loss: 0.7445 - val_accuracy: 0.8430\n",
            "Epoch 11/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.1062 - accuracy: 0.9678 - val_loss: 0.6398 - val_accuracy: 0.8234\n",
            "Epoch 12/30\n",
            "35/35 [==============================] - 7s 204ms/step - loss: 0.1165 - accuracy: 0.9642 - val_loss: 0.9751 - val_accuracy: 0.8340\n",
            "Epoch 13/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.0959 - accuracy: 0.9730 - val_loss: 0.9989 - val_accuracy: 0.8377\n",
            "Epoch 14/30\n",
            "35/35 [==============================] - 7s 204ms/step - loss: 0.0882 - accuracy: 0.9773 - val_loss: 0.8473 - val_accuracy: 0.8438\n",
            "Epoch 15/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.0686 - accuracy: 0.9826 - val_loss: 1.1860 - val_accuracy: 0.8302\n",
            "Epoch 16/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.0547 - accuracy: 0.9876 - val_loss: 1.3304 - val_accuracy: 0.8408\n",
            "Epoch 17/30\n",
            "35/35 [==============================] - 7s 202ms/step - loss: 0.0382 - accuracy: 0.9925 - val_loss: 1.5055 - val_accuracy: 0.8400\n",
            "Epoch 18/30\n",
            "35/35 [==============================] - 7s 205ms/step - loss: 0.0306 - accuracy: 0.9950 - val_loss: 1.5571 - val_accuracy: 0.8408\n",
            "Epoch 19/30\n",
            "35/35 [==============================] - 7s 204ms/step - loss: 0.0287 - accuracy: 0.9966 - val_loss: 1.5919 - val_accuracy: 0.8438\n",
            "Epoch 20/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.0281 - accuracy: 0.9970 - val_loss: 1.6884 - val_accuracy: 0.8445\n",
            "Epoch 21/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.0311 - accuracy: 0.9958 - val_loss: 1.7201 - val_accuracy: 0.8370\n",
            "Epoch 22/30\n",
            "35/35 [==============================] - 7s 204ms/step - loss: 0.0342 - accuracy: 0.9950 - val_loss: 1.5063 - val_accuracy: 0.8400\n",
            "Epoch 23/30\n",
            "35/35 [==============================] - 7s 204ms/step - loss: 0.0416 - accuracy: 0.9950 - val_loss: 1.5953 - val_accuracy: 0.8400\n",
            "Epoch 24/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.0351 - accuracy: 0.9963 - val_loss: 1.6878 - val_accuracy: 0.8340\n",
            "Epoch 25/30\n",
            "35/35 [==============================] - 7s 204ms/step - loss: 0.0345 - accuracy: 0.9950 - val_loss: 1.6790 - val_accuracy: 0.8385\n",
            "Epoch 26/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.0205 - accuracy: 0.9980 - val_loss: 1.6685 - val_accuracy: 0.8408\n",
            "Epoch 27/30\n",
            "35/35 [==============================] - 7s 201ms/step - loss: 0.0279 - accuracy: 0.9982 - val_loss: 1.8036 - val_accuracy: 0.8377\n",
            "Epoch 28/30\n",
            "35/35 [==============================] - 7s 202ms/step - loss: 0.0214 - accuracy: 0.9982 - val_loss: 1.7739 - val_accuracy: 0.8355\n",
            "Epoch 29/30\n",
            "35/35 [==============================] - 7s 202ms/step - loss: 0.0284 - accuracy: 0.9978 - val_loss: 1.7810 - val_accuracy: 0.8430\n",
            "Epoch 30/30\n",
            "35/35 [==============================] - 8s 217ms/step - loss: 0.0199 - accuracy: 0.9983 - val_loss: 2.7497 - val_accuracy: 0.7713\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lQcytuBdaWVp",
        "outputId": "a429ecd2-e3a4-4bd3-a255-72676eb557db"
      },
      "source": [
        "results = model.evaluate([x_test_int_np,x_test_masks_np], y_test_onehot)\n",
        "print(results)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "194/194 [==============================] - 8s 31ms/step - loss: 1.3703 - accuracy: 0.8784\n",
            "[1.370295524597168, 0.8784155249595642]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ohq4wzjDuIe3"
      },
      "source": [
        "Here for model 1 we are getting 87% accuracy. The accuracy we are getting for model 1 is more than the accuracy we were getting in lab5. The accuracy in the model1 is higher because here we are using DistilBert transfomer which is 60% faster and 40% less parameters than BERT and maintaining 95% efficiency of BERT which makes this models accruacy higher than model of lab5 because there we are using keras layer."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jwHxHiIhrm84"
      },
      "source": [
        "In your report, show the accuracy you achieve, and compare it to what you achieved in Lab 5. Is it better or worse, and why?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vdZ4nl08vp9A"
      },
      "source": [
        "\n",
        "## Model 2: Neural bag of words using BERT"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2gyCwXFj_R5w"
      },
      "source": [
        "In this model, we take the NBOW classifier from lab 4 (model3-1) and integrate BERT. Instead of averaging over word2vec or GloVe word vectors, we are averaging over the embedding representations produced by BERT - but otherwise, the classifier is the same.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DStlnRQRf-4v"
      },
      "source": [
        "class GlobalAveragePooling1DMasked(GlobalAveragePooling1D):\n",
        "    def call(self, x, mask=None):\n",
        "        if mask != None:\n",
        "            return K.sum(x, axis=1) / K.sum(mask, axis=1)\n",
        "        else:\n",
        "            return super().call(x)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8fTwmYDvNEyT"
      },
      "source": [
        "from transformers import TFDistilBertModel, DistilBertConfig\n",
        "\n",
        "def get_BERT_layer():\n",
        "  distil_bert = 'distilbert-base-uncased'\n",
        "  config = DistilBertConfig(dropout=0.2, attention_dropout=0.2)\n",
        "  config.output_hidden_states = False\n",
        "  return TFDistilBertModel.from_pretrained(distil_bert, config = config)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VICS9rY8C7KH",
        "outputId": "152d4d23-ea59-412f-8c02-ce78cf8e3a18"
      },
      "source": [
        "\n",
        "hdepth=16\n",
        "MAX_SEQUENCE_LENGTH = 128\n",
        "EMBED_SIZE=100\n",
        "\n",
        "\n",
        "def create_bag_of_words_BERT():\n",
        "  input_ids_in = tf.keras.layers.Input(shape=(128,), name='input_token', dtype='int32')\n",
        "  input_masks_in = tf.keras.layers.Input(shape=(128,), name='masked_token', dtype='int32') \n",
        "\n",
        "  bert_embeddings = get_BERT_layer()\n",
        "  embedded_sent = bert_embeddings(input_ids_in, attention_mask=input_masks_in)[0]\n",
        "\n",
        "  pooled_sent=GlobalAveragePooling1DMasked()(embedded_sent)\n",
        "  hidden_output=Dense(hdepth,input_shape=(MAX_SEQUENCE_LENGTH,EMBED_SIZE),activation='sigmoid',kernel_initializer='glorot_uniform')(pooled_sent) # Sigmoid\n",
        "  label=Dense(1,input_shape=(hdepth,),activation='sigmoid',kernel_initializer='glorot_uniform')(hidden_output)\n",
        "  return Model(inputs=[input_ids_in,input_masks_in], outputs=[label],name='Model2_BERT')\n",
        "\n",
        "use_tpu = True\n",
        "if use_tpu:\n",
        "  # Create distribution strategy\n",
        "  tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
        "  tf.config.experimental_connect_to_cluster(tpu)\n",
        "  tf.tpu.experimental.initialize_tpu_system(tpu)\n",
        "  strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
        "\n",
        "  # Create model\n",
        "  with strategy.scope():\n",
        "    model2 = create_bag_of_words_BERT()\n",
        "    optimizer2 = keras.optimizers.Adam(lr=5e-5)\n",
        "    model2.compile(optimizer=optimizer2, loss='binary_crossentropy', metrics=['accuracy'])\n",
        "else:\n",
        "  model2 = create_bag_of_words_BERT()\n",
        "  model2.compile(optimizer='adam',\n",
        "              loss='binary_crossentropy',\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "model2.summary() \n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:TPU system grpc://10.113.134.202:8470 has already been initialized. Reinitializing the TPU can cause previously created variables on TPU to be lost.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:TPU system grpc://10.113.134.202:8470 has already been initialized. Reinitializing the TPU can cause previously created variables on TPU to be lost.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.113.134.202:8470\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.113.134.202:8470\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n",
            "WARNING:absl:`tf.distribute.experimental.TPUStrategy` is deprecated, please use  the non experimental symbol `tf.distribute.TPUStrategy` instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n",
            "Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertModel: ['vocab_layer_norm', 'vocab_projector', 'vocab_transform', 'activation_13']\n",
            "- This IS expected if you are initializing TFDistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFDistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the layers of TFDistilBertModel were initialized from the model checkpoint at distilbert-base-uncased.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFDistilBertModel for predictions without further training.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Model: \"Model2_BERT\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_token (InputLayer)        [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "masked_token (InputLayer)       [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "tf_distil_bert_model (TFDistilB TFBaseModelOutput(la 66362880    input_token[0][0]                \n",
            "                                                                 masked_token[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling1d_masked (None, 768)          0           tf_distil_bert_model[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 16)           12304       global_average_pooling1d_masked[0\n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 1)            17          dense[0][0]                      \n",
            "==================================================================================================\n",
            "Total params: 66,375,201\n",
            "Trainable params: 66,375,201\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S0SbsCsxF1zi",
        "outputId": "1af20f0e-4e37-4fa1-d17c-7cddfdf91905"
      },
      "source": [
        "\n",
        "history = model2.fit([x_train_int_np,x_train_masks_np],\n",
        "                    y_train,\n",
        "                    epochs=30,\n",
        "                    batch_size=512,\n",
        "                    validation_data=([x_dev_int_np,x_dev_masks_np], y_dev),\n",
        "                    verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/30\n",
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "35/35 [==============================] - ETA: 0s - loss: 0.5273 - accuracy: 0.7354WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r35/35 [==============================] - 92s 1s/step - loss: 0.5247 - accuracy: 0.7376 - val_loss: 0.3672 - val_accuracy: 0.8385\n",
            "Epoch 2/30\n",
            "35/35 [==============================] - 7s 202ms/step - loss: 0.2911 - accuracy: 0.8979 - val_loss: 0.3414 - val_accuracy: 0.8596\n",
            "Epoch 3/30\n",
            "35/35 [==============================] - 7s 202ms/step - loss: 0.2294 - accuracy: 0.9351 - val_loss: 0.3728 - val_accuracy: 0.8521\n",
            "Epoch 4/30\n",
            "35/35 [==============================] - 7s 202ms/step - loss: 0.1981 - accuracy: 0.9529 - val_loss: 0.3780 - val_accuracy: 0.8498\n",
            "Epoch 5/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.1752 - accuracy: 0.9636 - val_loss: 0.4069 - val_accuracy: 0.8453\n",
            "Epoch 6/30\n",
            "35/35 [==============================] - 7s 201ms/step - loss: 0.1595 - accuracy: 0.9716 - val_loss: 0.3807 - val_accuracy: 0.8604\n",
            "Epoch 7/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.1532 - accuracy: 0.9742 - val_loss: 0.4218 - val_accuracy: 0.8438\n",
            "Epoch 8/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.1406 - accuracy: 0.9793 - val_loss: 0.4115 - val_accuracy: 0.8498\n",
            "Epoch 9/30\n",
            "35/35 [==============================] - 7s 202ms/step - loss: 0.1436 - accuracy: 0.9779 - val_loss: 0.4491 - val_accuracy: 0.8340\n",
            "Epoch 10/30\n",
            "35/35 [==============================] - 7s 202ms/step - loss: 0.1344 - accuracy: 0.9817 - val_loss: 0.4198 - val_accuracy: 0.8491\n",
            "Epoch 11/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.1299 - accuracy: 0.9835 - val_loss: 0.4424 - val_accuracy: 0.8408\n",
            "Epoch 12/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.1309 - accuracy: 0.9824 - val_loss: 0.4491 - val_accuracy: 0.8430\n",
            "Epoch 13/30\n",
            "35/35 [==============================] - 7s 204ms/step - loss: 0.1301 - accuracy: 0.9815 - val_loss: 0.4303 - val_accuracy: 0.8460\n",
            "Epoch 14/30\n",
            "35/35 [==============================] - 8s 219ms/step - loss: 0.1261 - accuracy: 0.9839 - val_loss: 0.4746 - val_accuracy: 0.8347\n",
            "Epoch 15/30\n",
            "35/35 [==============================] - 7s 205ms/step - loss: 0.1203 - accuracy: 0.9853 - val_loss: 0.4452 - val_accuracy: 0.8445\n",
            "Epoch 16/30\n",
            "35/35 [==============================] - 7s 204ms/step - loss: 0.1190 - accuracy: 0.9861 - val_loss: 0.4121 - val_accuracy: 0.8611\n",
            "Epoch 17/30\n",
            "35/35 [==============================] - 7s 204ms/step - loss: 0.1132 - accuracy: 0.9882 - val_loss: 0.4194 - val_accuracy: 0.8528\n",
            "Epoch 18/30\n",
            "35/35 [==============================] - 7s 202ms/step - loss: 0.1107 - accuracy: 0.9891 - val_loss: 0.4559 - val_accuracy: 0.8408\n",
            "Epoch 19/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.1082 - accuracy: 0.9894 - val_loss: 0.4326 - val_accuracy: 0.8521\n",
            "Epoch 20/30\n",
            "35/35 [==============================] - 7s 204ms/step - loss: 0.1119 - accuracy: 0.9873 - val_loss: 0.4301 - val_accuracy: 0.8536\n",
            "Epoch 21/30\n",
            "35/35 [==============================] - 7s 202ms/step - loss: 0.1093 - accuracy: 0.9883 - val_loss: 0.4569 - val_accuracy: 0.8423\n",
            "Epoch 22/30\n",
            "35/35 [==============================] - 7s 204ms/step - loss: 0.1059 - accuracy: 0.9896 - val_loss: 0.4473 - val_accuracy: 0.8445\n",
            "Epoch 23/30\n",
            "35/35 [==============================] - 7s 203ms/step - loss: 0.1060 - accuracy: 0.9890 - val_loss: 0.4414 - val_accuracy: 0.8506\n",
            "Epoch 24/30\n",
            "35/35 [==============================] - 7s 206ms/step - loss: 0.1015 - accuracy: 0.9906 - val_loss: 0.4474 - val_accuracy: 0.8506\n",
            "Epoch 25/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.1010 - accuracy: 0.9904 - val_loss: 0.4458 - val_accuracy: 0.8491\n",
            "Epoch 26/30\n",
            "35/35 [==============================] - 7s 212ms/step - loss: 0.1079 - accuracy: 0.9869 - val_loss: 0.4435 - val_accuracy: 0.8483\n",
            "Epoch 27/30\n",
            "35/35 [==============================] - 7s 214ms/step - loss: 0.1033 - accuracy: 0.9886 - val_loss: 0.4375 - val_accuracy: 0.8498\n",
            "Epoch 28/30\n",
            "35/35 [==============================] - 7s 205ms/step - loss: 0.1013 - accuracy: 0.9895 - val_loss: 0.4633 - val_accuracy: 0.8400\n",
            "Epoch 29/30\n",
            "35/35 [==============================] - 7s 205ms/step - loss: 0.0989 - accuracy: 0.9899 - val_loss: 0.4688 - val_accuracy: 0.8408\n",
            "Epoch 30/30\n",
            "35/35 [==============================] - 7s 205ms/step - loss: 0.0976 - accuracy: 0.9903 - val_loss: 0.4266 - val_accuracy: 0.8589\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rs0_vvG6UQtv",
        "outputId": "8f2a0996-d976-4821-9e64-473019ede464"
      },
      "source": [
        "results = model2.evaluate([x_test_int_np,x_test_masks_np], y_test)\n",
        "print(results)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "194/194 [==============================] - 8s 31ms/step - loss: 0.3960 - accuracy: 0.8572\n",
            "[0.39597558975219727, 0.8572352528572083]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5UQpWd9-rz1I"
      },
      "source": [
        "Here for model2 We are getting 85.7% accuracy which is significantly more than the model 1 of lab5. In model1 lab5 we were getting around 70% of the accuracy. The classifier here is same as we used in lab4 but here we are using the bert embeddings rather thant glove vectors and this bert transformer embedding makes a significant different in the accuracy of the two models."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IuHV_OiSrm88"
      },
      "source": [
        "In your report, show the accuracy you achieve, and compare it to what you achieved with Model 1 and in Lab 5. Is it better or worse, and why?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "awOphcCnhEwv"
      },
      "source": [
        "## Model 3: CNN or LSTM with BERT"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E5codSzohQ_9"
      },
      "source": [
        "Please following model2 to finish a CNN or LSTM model with BERT. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JMiiWhW4hPRA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "99e3fe03-5168-4ff5-86d1-4d1c15d015a1"
      },
      "source": [
        "# your code goes here\n",
        "#lstm model\n",
        "hdepth=16\n",
        "MAX_SEQUENCE_LENGTH = 128\n",
        "EMBED_SIZE=100\n",
        "\n",
        "\n",
        "def create_bag_of_words_BERT():\n",
        "  input_ids_in = tf.keras.layers.Input(shape=(128,), name='input_token', dtype='int32')\n",
        "  input_masks_in = tf.keras.layers.Input(shape=(128,), name='masked_token', dtype='int32') \n",
        "\n",
        "  bert_embeddings = get_BERT_layer()\n",
        "  embedded_sent = bert_embeddings(input_ids_in, attention_mask=input_masks_in)[0]\n",
        "  lstm = LSTM(100, return_sequences= False)(embedded_sent)\n",
        "  #pooled_sent=GlobalAveragePooling1DMasked()(embedded_sent)\n",
        "  hidden_output=Dense(hdepth,input_shape=(MAX_SEQUENCE_LENGTH,EMBED_SIZE),activation='sigmoid',kernel_initializer='glorot_uniform')(lstm) # Sigmoid\n",
        "  label=Dense(1,input_shape=(hdepth,),activation='sigmoid',kernel_initializer='glorot_uniform')(hidden_output)\n",
        "  return Model(inputs=[input_ids_in,input_masks_in], outputs=[label],name='Model2_BERT')\n",
        "\n",
        "use_tpu = True\n",
        "if use_tpu:\n",
        "  # Create distribution strategy\n",
        "  tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
        "  tf.config.experimental_connect_to_cluster(tpu)\n",
        "  tf.tpu.experimental.initialize_tpu_system(tpu)\n",
        "  strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
        "\n",
        "  # Create model\n",
        "  with strategy.scope():\n",
        "    model3 = create_bag_of_words_BERT()\n",
        "    optimizer3 = keras.optimizers.Adam(lr=5e-5)\n",
        "    model3.compile(optimizer=optimizer3, loss='binary_crossentropy', metrics=['accuracy'])\n",
        "else:\n",
        "  model3 = create_bag_of_words_BERT()\n",
        "  model.compile(optimizer='adam',\n",
        "              loss='binary_crossentropy',\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "model3.summary() \n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:TPU system grpc://10.113.134.202:8470 has already been initialized. Reinitializing the TPU can cause previously created variables on TPU to be lost.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:TPU system grpc://10.113.134.202:8470 has already been initialized. Reinitializing the TPU can cause previously created variables on TPU to be lost.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.113.134.202:8470\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.113.134.202:8470\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n",
            "WARNING:absl:`tf.distribute.experimental.TPUStrategy` is deprecated, please use  the non experimental symbol `tf.distribute.TPUStrategy` instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n",
            "Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertModel: ['vocab_layer_norm', 'vocab_projector', 'vocab_transform', 'activation_13']\n",
            "- This IS expected if you are initializing TFDistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFDistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the layers of TFDistilBertModel were initialized from the model checkpoint at distilbert-base-uncased.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFDistilBertModel for predictions without further training.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Model: \"Model2_BERT\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_token (InputLayer)        [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "masked_token (InputLayer)       [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "tf_distil_bert_model_1 (TFDisti TFBaseModelOutput(la 66362880    input_token[0][0]                \n",
            "                                                                 masked_token[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "lstm (LSTM)                     (None, 100)          347600      tf_distil_bert_model_1[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 16)           1616        lstm[0][0]                       \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 1)            17          dense_2[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 66,712,113\n",
            "Trainable params: 66,712,113\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m25PAKx-I5KQ"
      },
      "source": [
        "Following model2 I have created a lstm model, adding a lstm layer in model 2."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JL2gwNaO0Wd8",
        "outputId": "61530f99-5e1a-4df4-9724-de7ac6f1a2af"
      },
      "source": [
        "\n",
        "history = model3.fit([x_train_int_np,x_train_masks_np],\n",
        "                    y_train,\n",
        "                    epochs=30,\n",
        "                    batch_size=512,\n",
        "                    validation_data=([x_dev_int_np,x_dev_masks_np], y_dev),\n",
        "                    verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/30\n",
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "35/35 [==============================] - ETA: 0s - loss: 0.6716 - accuracy: 0.5893WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r35/35 [==============================] - 96s 1s/step - loss: 0.6690 - accuracy: 0.5930 - val_loss: 0.5686 - val_accuracy: 0.7442\n",
            "Epoch 2/30\n",
            "35/35 [==============================] - 7s 213ms/step - loss: 0.5135 - accuracy: 0.7894 - val_loss: 0.4496 - val_accuracy: 0.8302\n",
            "Epoch 3/30\n",
            "35/35 [==============================] - 7s 211ms/step - loss: 0.4119 - accuracy: 0.8654 - val_loss: 0.4190 - val_accuracy: 0.8370\n",
            "Epoch 4/30\n",
            "35/35 [==============================] - 7s 211ms/step - loss: 0.3440 - accuracy: 0.9039 - val_loss: 0.3916 - val_accuracy: 0.8732\n",
            "Epoch 5/30\n",
            "35/35 [==============================] - 7s 210ms/step - loss: 0.2969 - accuracy: 0.9332 - val_loss: 0.3830 - val_accuracy: 0.8687\n",
            "Epoch 6/30\n",
            "35/35 [==============================] - 7s 213ms/step - loss: 0.2697 - accuracy: 0.9471 - val_loss: 0.4007 - val_accuracy: 0.8506\n",
            "Epoch 7/30\n",
            "35/35 [==============================] - 7s 211ms/step - loss: 0.2564 - accuracy: 0.9508 - val_loss: 0.4241 - val_accuracy: 0.8574\n",
            "Epoch 8/30\n",
            "35/35 [==============================] - 7s 209ms/step - loss: 0.2312 - accuracy: 0.9637 - val_loss: 0.3963 - val_accuracy: 0.8604\n",
            "Epoch 9/30\n",
            "35/35 [==============================] - 7s 212ms/step - loss: 0.2151 - accuracy: 0.9698 - val_loss: 0.4075 - val_accuracy: 0.8611\n",
            "Epoch 10/30\n",
            "35/35 [==============================] - 7s 211ms/step - loss: 0.2060 - accuracy: 0.9735 - val_loss: 0.4156 - val_accuracy: 0.8581\n",
            "Epoch 11/30\n",
            "35/35 [==============================] - 7s 212ms/step - loss: 0.1973 - accuracy: 0.9760 - val_loss: 0.4366 - val_accuracy: 0.8543\n",
            "Epoch 12/30\n",
            "35/35 [==============================] - 8s 234ms/step - loss: 0.2011 - accuracy: 0.9741 - val_loss: 0.4299 - val_accuracy: 0.8551\n",
            "Epoch 13/30\n",
            "35/35 [==============================] - 7s 212ms/step - loss: 0.1955 - accuracy: 0.9753 - val_loss: 0.4572 - val_accuracy: 0.8543\n",
            "Epoch 14/30\n",
            "35/35 [==============================] - 7s 212ms/step - loss: 0.1884 - accuracy: 0.9770 - val_loss: 0.4194 - val_accuracy: 0.8581\n",
            "Epoch 15/30\n",
            "35/35 [==============================] - 7s 212ms/step - loss: 0.1804 - accuracy: 0.9795 - val_loss: 0.4366 - val_accuracy: 0.8581\n",
            "Epoch 16/30\n",
            "35/35 [==============================] - 7s 211ms/step - loss: 0.1794 - accuracy: 0.9773 - val_loss: 0.4332 - val_accuracy: 0.8611\n",
            "Epoch 17/30\n",
            "35/35 [==============================] - 7s 212ms/step - loss: 0.1733 - accuracy: 0.9797 - val_loss: 0.4204 - val_accuracy: 0.8574\n",
            "Epoch 18/30\n",
            "35/35 [==============================] - 7s 212ms/step - loss: 0.1689 - accuracy: 0.9810 - val_loss: 0.4680 - val_accuracy: 0.8521\n",
            "Epoch 19/30\n",
            "35/35 [==============================] - 7s 212ms/step - loss: 0.1660 - accuracy: 0.9809 - val_loss: 0.4152 - val_accuracy: 0.8596\n",
            "Epoch 20/30\n",
            "35/35 [==============================] - 7s 211ms/step - loss: 0.1662 - accuracy: 0.9809 - val_loss: 0.4261 - val_accuracy: 0.8536\n",
            "Epoch 21/30\n",
            "35/35 [==============================] - 7s 212ms/step - loss: 0.1614 - accuracy: 0.9814 - val_loss: 0.4271 - val_accuracy: 0.8528\n",
            "Epoch 22/30\n",
            "35/35 [==============================] - 7s 212ms/step - loss: 0.1581 - accuracy: 0.9821 - val_loss: 0.4167 - val_accuracy: 0.8596\n",
            "Epoch 23/30\n",
            "35/35 [==============================] - 8s 216ms/step - loss: 0.1518 - accuracy: 0.9848 - val_loss: 0.4659 - val_accuracy: 0.8445\n",
            "Epoch 24/30\n",
            "35/35 [==============================] - 8s 219ms/step - loss: 0.1505 - accuracy: 0.9841 - val_loss: 0.4397 - val_accuracy: 0.8558\n",
            "Epoch 25/30\n",
            "35/35 [==============================] - 8s 217ms/step - loss: 0.1470 - accuracy: 0.9851 - val_loss: 0.4203 - val_accuracy: 0.8536\n",
            "Epoch 26/30\n",
            "35/35 [==============================] - 8s 219ms/step - loss: 0.1437 - accuracy: 0.9865 - val_loss: 0.4472 - val_accuracy: 0.8506\n",
            "Epoch 27/30\n",
            "35/35 [==============================] - 8s 216ms/step - loss: 0.1461 - accuracy: 0.9842 - val_loss: 0.4492 - val_accuracy: 0.8468\n",
            "Epoch 28/30\n",
            "35/35 [==============================] - 7s 213ms/step - loss: 0.1401 - accuracy: 0.9863 - val_loss: 0.4433 - val_accuracy: 0.8468\n",
            "Epoch 29/30\n",
            "35/35 [==============================] - 7s 212ms/step - loss: 0.1384 - accuracy: 0.9864 - val_loss: 0.4764 - val_accuracy: 0.8483\n",
            "Epoch 30/30\n",
            "35/35 [==============================] - 7s 213ms/step - loss: 0.1434 - accuracy: 0.9835 - val_loss: 0.4256 - val_accuracy: 0.8551\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J0A13a370htO",
        "outputId": "80baa66e-1f1b-47fb-c36b-80f8543a7df0"
      },
      "source": [
        "results = model3.evaluate([x_test_int_np,x_test_masks_np], y_test)\n",
        "print(results)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "194/194 [==============================] - 9s 33ms/step - loss: 0.4059 - accuracy: 0.8451\n",
            "[0.4058905243873596, 0.8451091051101685]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6Xw6YRbqJHfw"
      },
      "source": [
        "Here we can see that we are getting 84.5% accuracy which is slightly less than the model2."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kwcLk_kH1t9k",
        "outputId": "e95e6957-a67d-4866-998e-6404b32cfe0c"
      },
      "source": [
        "# your code goes here\n",
        "#CNN model\n",
        "hdepth=16\n",
        "MAX_SEQUENCE_LENGTH = 128\n",
        "EMBED_SIZE=100\n",
        "\n",
        "\n",
        "def create_bag_of_words_BERT():\n",
        "  input_ids_in = tf.keras.layers.Input(shape=(128,), name='input_token', dtype='int32')\n",
        "  input_masks_in = tf.keras.layers.Input(shape=(128,), name='masked_token', dtype='int32') \n",
        "\n",
        "  bert_embeddings = get_BERT_layer()\n",
        "  embedded_sent = bert_embeddings(input_ids_in, attention_mask=input_masks_in)[0]\n",
        "  cnn = keras.layers.Conv1D(100, 6)(embedded_sent)\n",
        "  pooled_sent=GlobalAveragePooling1DMasked()(cnn)\n",
        "  hidden_output=Dense(hdepth,input_shape=(MAX_SEQUENCE_LENGTH,EMBED_SIZE),activation='sigmoid',kernel_initializer='glorot_uniform')(pooled_sent) # Sigmoid\n",
        "  label=Dense(1,input_shape=(hdepth,),activation='sigmoid',kernel_initializer='glorot_uniform')(hidden_output)\n",
        "  return Model(inputs=[input_ids_in,input_masks_in], outputs=[label],name='Model2_BERT')\n",
        "\n",
        "use_tpu = True\n",
        "if use_tpu:\n",
        "  # Create distribution strategy\n",
        "  tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
        "  tf.config.experimental_connect_to_cluster(tpu)\n",
        "  tf.tpu.experimental.initialize_tpu_system(tpu)\n",
        "  strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
        "\n",
        "  # Create model\n",
        "  with strategy.scope():\n",
        "    model4 = create_bag_of_words_BERT()\n",
        "    optimizer4 = keras.optimizers.Adam(lr=5e-5)\n",
        "    model4.compile(optimizer=optimizer4, loss='binary_crossentropy', metrics=['accuracy'])\n",
        "else:\n",
        "  model4 = create_bag_of_words_BERT()\n",
        "  model.compile(optimizer='adam',\n",
        "              loss='binary_crossentropy',\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "model4.summary() \n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:TPU system grpc://10.113.134.202:8470 has already been initialized. Reinitializing the TPU can cause previously created variables on TPU to be lost.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:TPU system grpc://10.113.134.202:8470 has already been initialized. Reinitializing the TPU can cause previously created variables on TPU to be lost.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.113.134.202:8470\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.113.134.202:8470\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n",
            "WARNING:absl:`tf.distribute.experimental.TPUStrategy` is deprecated, please use  the non experimental symbol `tf.distribute.TPUStrategy` instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n",
            "Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertModel: ['vocab_layer_norm', 'vocab_projector', 'vocab_transform', 'activation_13']\n",
            "- This IS expected if you are initializing TFDistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFDistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the layers of TFDistilBertModel were initialized from the model checkpoint at distilbert-base-uncased.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFDistilBertModel for predictions without further training.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Model: \"Model2_BERT\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_token (InputLayer)        [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "masked_token (InputLayer)       [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "tf_distil_bert_model_2 (TFDisti TFBaseModelOutput(la 66362880    input_token[0][0]                \n",
            "                                                                 masked_token[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "conv1d (Conv1D)                 (None, 123, 100)     460900      tf_distil_bert_model_2[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling1d_masked (None, 100)          0           conv1d[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "dense_4 (Dense)                 (None, 16)           1616        global_average_pooling1d_masked_1\n",
            "__________________________________________________________________________________________________\n",
            "dense_5 (Dense)                 (None, 1)            17          dense_4[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 66,825,413\n",
            "Trainable params: 66,825,413\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0UUhhi5vJSW3"
      },
      "source": [
        "Now after adding a lstm layer on model 2 finding that there is not much change in the accuracy, Now i have build a CNN model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cudvRicr1uIJ",
        "outputId": "20274329-75bf-447e-bf7c-5bf583c592b9"
      },
      "source": [
        "\n",
        "history = model4.fit([x_train_int_np,x_train_masks_np],\n",
        "                    y_train,\n",
        "                    epochs=30,\n",
        "                    batch_size=512,\n",
        "                    validation_data=([x_dev_int_np,x_dev_masks_np], y_dev),\n",
        "                    verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/30\n",
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "35/35 [==============================] - ETA: 0s - loss: 0.4618 - accuracy: 0.8099WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r35/35 [==============================] - 96s 1s/step - loss: 0.4594 - accuracy: 0.8109 - val_loss: 0.3413 - val_accuracy: 0.8551\n",
            "Epoch 2/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.2635 - accuracy: 0.9021 - val_loss: 0.3358 - val_accuracy: 0.8506\n",
            "Epoch 3/30\n",
            "35/35 [==============================] - 7s 206ms/step - loss: 0.2054 - accuracy: 0.9365 - val_loss: 0.3434 - val_accuracy: 0.8558\n",
            "Epoch 4/30\n",
            "35/35 [==============================] - 7s 205ms/step - loss: 0.1783 - accuracy: 0.9540 - val_loss: 0.3597 - val_accuracy: 0.8536\n",
            "Epoch 5/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.1567 - accuracy: 0.9644 - val_loss: 0.3805 - val_accuracy: 0.8528\n",
            "Epoch 6/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.1399 - accuracy: 0.9731 - val_loss: 0.4087 - val_accuracy: 0.8438\n",
            "Epoch 7/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.1403 - accuracy: 0.9720 - val_loss: 0.4003 - val_accuracy: 0.8377\n",
            "Epoch 8/30\n",
            "35/35 [==============================] - 7s 208ms/step - loss: 0.1223 - accuracy: 0.9802 - val_loss: 0.4322 - val_accuracy: 0.8279\n",
            "Epoch 9/30\n",
            "35/35 [==============================] - 7s 208ms/step - loss: 0.1198 - accuracy: 0.9815 - val_loss: 0.4070 - val_accuracy: 0.8445\n",
            "Epoch 10/30\n",
            "35/35 [==============================] - 7s 205ms/step - loss: 0.1171 - accuracy: 0.9818 - val_loss: 0.4081 - val_accuracy: 0.8430\n",
            "Epoch 11/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.1136 - accuracy: 0.9833 - val_loss: 0.4033 - val_accuracy: 0.8468\n",
            "Epoch 12/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.1100 - accuracy: 0.9845 - val_loss: 0.4093 - val_accuracy: 0.8400\n",
            "Epoch 13/30\n",
            "35/35 [==============================] - 7s 208ms/step - loss: 0.1046 - accuracy: 0.9866 - val_loss: 0.4237 - val_accuracy: 0.8355\n",
            "Epoch 14/30\n",
            "35/35 [==============================] - 7s 208ms/step - loss: 0.1031 - accuracy: 0.9878 - val_loss: 0.4411 - val_accuracy: 0.8370\n",
            "Epoch 15/30\n",
            "35/35 [==============================] - 7s 206ms/step - loss: 0.1034 - accuracy: 0.9866 - val_loss: 0.4128 - val_accuracy: 0.8551\n",
            "Epoch 16/30\n",
            "35/35 [==============================] - 7s 206ms/step - loss: 0.0958 - accuracy: 0.9898 - val_loss: 0.4323 - val_accuracy: 0.8392\n",
            "Epoch 17/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.1060 - accuracy: 0.9851 - val_loss: 0.4145 - val_accuracy: 0.8491\n",
            "Epoch 18/30\n",
            "35/35 [==============================] - 7s 206ms/step - loss: 0.0976 - accuracy: 0.9881 - val_loss: 0.4153 - val_accuracy: 0.8483\n",
            "Epoch 19/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.0979 - accuracy: 0.9873 - val_loss: 0.4227 - val_accuracy: 0.8423\n",
            "Epoch 20/30\n",
            "35/35 [==============================] - 7s 206ms/step - loss: 0.0920 - accuracy: 0.9903 - val_loss: 0.4329 - val_accuracy: 0.8438\n",
            "Epoch 21/30\n",
            "35/35 [==============================] - 7s 208ms/step - loss: 0.0963 - accuracy: 0.9880 - val_loss: 0.4266 - val_accuracy: 0.8408\n",
            "Epoch 22/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.0919 - accuracy: 0.9896 - val_loss: 0.4434 - val_accuracy: 0.8385\n",
            "Epoch 23/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.0893 - accuracy: 0.9902 - val_loss: 0.4425 - val_accuracy: 0.8430\n",
            "Epoch 24/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.0918 - accuracy: 0.9890 - val_loss: 0.4371 - val_accuracy: 0.8453\n",
            "Epoch 25/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.0878 - accuracy: 0.9895 - val_loss: 0.4456 - val_accuracy: 0.8400\n",
            "Epoch 26/30\n",
            "35/35 [==============================] - 7s 206ms/step - loss: 0.0861 - accuracy: 0.9902 - val_loss: 0.4464 - val_accuracy: 0.8400\n",
            "Epoch 27/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.0882 - accuracy: 0.9897 - val_loss: 0.4441 - val_accuracy: 0.8475\n",
            "Epoch 28/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.0867 - accuracy: 0.9899 - val_loss: 0.4522 - val_accuracy: 0.8400\n",
            "Epoch 29/30\n",
            "35/35 [==============================] - 7s 207ms/step - loss: 0.0834 - accuracy: 0.9906 - val_loss: 0.4565 - val_accuracy: 0.8325\n",
            "Epoch 30/30\n",
            "35/35 [==============================] - 7s 208ms/step - loss: 0.0829 - accuracy: 0.9908 - val_loss: 0.4558 - val_accuracy: 0.8408\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dXQeH2uf2LuL",
        "outputId": "6c7de147-b8c5-4a94-ddb1-05456c9db2e2"
      },
      "source": [
        "results = model4.evaluate([x_test_int_np,x_test_masks_np], y_test)\n",
        "print(results)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "194/194 [==============================] - 8s 31ms/step - loss: 0.4083 - accuracy: 0.8663\n",
            "[0.4082832336425781, 0.8662893772125244]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p4sSaQUSJhEp"
      },
      "source": [
        "Here we can see that CNN model gives the accuracy of 86.6% which is more comparative to model 2 and model2 with lstm layer. But it gives almost equal accuracy compare to model1. The accuracy has increased because here we are using a cnn layer whereas in the model2 we are using neural bag of words with BERT."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g8eXc6SGrm8-"
      },
      "source": [
        "In your report, show the accuracy you achieve, and compare it to what you achieved with Model 1 and Model 2. Is it better or worse, and why?"
      ]
    }
  ]
}