Transformers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "33c934ff057c4ee882294a4d8b5d08a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_322c6638cacc429ca0ec6356df2d16d8",
              "IPY_MODEL_55b6898f66c343f6bbc951c8034c6a14",
              "IPY_MODEL_8aabd47a2b884fecb183f82c87c37c2f"
            ],
            "layout": "IPY_MODEL_f1967b248be646978d25b37b19c189d0"
          }
        },
        "322c6638cacc429ca0ec6356df2d16d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a78558c8f6c24890b2da8ef4ffba7fbc",
            "placeholder": "​",
            "style": "IPY_MODEL_45fddca2236c4f399d5a6f0a24b7ffde",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "55b6898f66c343f6bbc951c8034c6a14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7d7c703f325b4c479e45ff367774468f",
            "max": 48,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_407c44f6938349a1b2da55279656b7b4",
            "value": 48
          }
        },
        "8aabd47a2b884fecb183f82c87c37c2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4e2090ce05ed4fb49e09b8ceb42dce09",
            "placeholder": "​",
            "style": "IPY_MODEL_7a8dc70a43264e89bf6611832e1e0493",
            "value": " 48.0/48.0 [00:00&lt;00:00, 889B/s]"
          }
        },
        "f1967b248be646978d25b37b19c189d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a78558c8f6c24890b2da8ef4ffba7fbc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "45fddca2236c4f399d5a6f0a24b7ffde": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7d7c703f325b4c479e45ff367774468f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "407c44f6938349a1b2da55279656b7b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4e2090ce05ed4fb49e09b8ceb42dce09": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7a8dc70a43264e89bf6611832e1e0493": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ceaaaf35614b4df4afd9a8b6b5964cae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_63296e04a09c4326888a880af0b47f45",
              "IPY_MODEL_ce0f2d10ca3a4b9d89eb68a08356c61a",
              "IPY_MODEL_e06b1ac6facb464fa093c1049ee2b986"
            ],
            "layout": "IPY_MODEL_50faee8731874d4783c01f8fd2afcc61"
          }
        },
        "63296e04a09c4326888a880af0b47f45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5d54676ba2784cb4a9b543cae8a23780",
            "placeholder": "​",
            "style": "IPY_MODEL_1949d7f5bafd48cb8ef27c4e0901b227",
            "value": "vocab.txt: 100%"
          }
        },
        "ce0f2d10ca3a4b9d89eb68a08356c61a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3716b098d44e4515b092ee90f39dc735",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6ce011ede20c4bbe9df3edcf2aeddbb1",
            "value": 231508
          }
        },
        "e06b1ac6facb464fa093c1049ee2b986": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_782faed5bbda4757b5875ed3592a8465",
            "placeholder": "​",
            "style": "IPY_MODEL_93fc262be1184457b0820cc4e4bbe388",
            "value": " 232k/232k [00:00&lt;00:00, 2.72MB/s]"
          }
        },
        "50faee8731874d4783c01f8fd2afcc61": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5d54676ba2784cb4a9b543cae8a23780": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1949d7f5bafd48cb8ef27c4e0901b227": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3716b098d44e4515b092ee90f39dc735": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6ce011ede20c4bbe9df3edcf2aeddbb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "782faed5bbda4757b5875ed3592a8465": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "93fc262be1184457b0820cc4e4bbe388": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a91a61753a8c4555812a635e7e8a6ed5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a8c18751bbe94cc8ab5264c30ffbd01c",
              "IPY_MODEL_74fa3150483c48da9a63b06602250f88",
              "IPY_MODEL_481967cb0649479183832a6506af35b8"
            ],
            "layout": "IPY_MODEL_b7d505edf2744176b8adaecebb512194"
          }
        },
        "a8c18751bbe94cc8ab5264c30ffbd01c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ab2759518b90459181221d142233ffcd",
            "placeholder": "​",
            "style": "IPY_MODEL_f6a7e926af1b44b79a5840c39e1ec3b4",
            "value": "tokenizer.json: 100%"
          }
        },
        "74fa3150483c48da9a63b06602250f88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_84c3b9adc4e848c3851e9b866ad90bd7",
            "max": 466062,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ec47a105805947fb88196be17a39a7c3",
            "value": 466062
          }
        },
        "481967cb0649479183832a6506af35b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_40170304fe1a46d9930fd1e4913e0efd",
            "placeholder": "​",
            "style": "IPY_MODEL_0f884d80f1204955b185a89a71b248a2",
            "value": " 466k/466k [00:00&lt;00:00, 4.19MB/s]"
          }
        },
        "b7d505edf2744176b8adaecebb512194": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ab2759518b90459181221d142233ffcd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f6a7e926af1b44b79a5840c39e1ec3b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "84c3b9adc4e848c3851e9b866ad90bd7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ec47a105805947fb88196be17a39a7c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "40170304fe1a46d9930fd1e4913e0efd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0f884d80f1204955b185a89a71b248a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "34b88f3f0ac7409a9a28a0ca3640b654": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_06112d4e32034dc3a1de278df3d3c488",
              "IPY_MODEL_ee5bfc02c18342a3a1c4718871a979d8",
              "IPY_MODEL_e86c4cbe362742f280e3b25b16e936e0"
            ],
            "layout": "IPY_MODEL_a6f017546b64402e9ad363773d9e1c10"
          }
        },
        "06112d4e32034dc3a1de278df3d3c488": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b8ccc02294f745f8aad3ebe5f7e5622a",
            "placeholder": "​",
            "style": "IPY_MODEL_29abaad8458e40309e1326176beb12f5",
            "value": "config.json: 100%"
          }
        },
        "ee5bfc02c18342a3a1c4718871a979d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_13013f89afdd4a8195ca2a26433cfbda",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0487c1da1767418292ad5dccb300c1de",
            "value": 570
          }
        },
        "e86c4cbe362742f280e3b25b16e936e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_febfc7ccfd3544fba5d25d6ccc691c04",
            "placeholder": "​",
            "style": "IPY_MODEL_731d7eee9f1a4099a47d4fdc3ea7048b",
            "value": " 570/570 [00:00&lt;00:00, 10.2kB/s]"
          }
        },
        "a6f017546b64402e9ad363773d9e1c10": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b8ccc02294f745f8aad3ebe5f7e5622a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "29abaad8458e40309e1326176beb12f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "13013f89afdd4a8195ca2a26433cfbda": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0487c1da1767418292ad5dccb300c1de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "febfc7ccfd3544fba5d25d6ccc691c04": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "731d7eee9f1a4099a47d4fdc3ea7048b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c4d2e924d1344186ba7add7b2facaf95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ea0f792474f24efe833dd68b2cf35f72",
              "IPY_MODEL_e4ca83b501544c3cb42daeee40255b26",
              "IPY_MODEL_f8d71f1116da42e2b02e458454665663"
            ],
            "layout": "IPY_MODEL_8697f42fdfed49a0935080467402e738"
          }
        },
        "ea0f792474f24efe833dd68b2cf35f72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0a1c9ddc24334e1bb6fd7ac915b45978",
            "placeholder": "​",
            "style": "IPY_MODEL_2042a701e05440faad772ee7e688f247",
            "value": "model.safetensors: 100%"
          }
        },
        "e4ca83b501544c3cb42daeee40255b26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_39d938eef56e438ba5ba6ad544f40f66",
            "max": 440449768,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ce2c1b320ce743459d49701891fc5800",
            "value": 440449768
          }
        },
        "f8d71f1116da42e2b02e458454665663": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_62bc1930a4df4a8eb9ff5283478a29ee",
            "placeholder": "​",
            "style": "IPY_MODEL_ab4ef4cdd23f4007ab84356dddae558b",
            "value": " 440M/440M [00:07&lt;00:00, 49.2MB/s]"
          }
        },
        "8697f42fdfed49a0935080467402e738": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0a1c9ddc24334e1bb6fd7ac915b45978": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2042a701e05440faad772ee7e688f247": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "39d938eef56e438ba5ba6ad544f40f66": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce2c1b320ce743459d49701891fc5800": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "62bc1930a4df4a8eb9ff5283478a29ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ab4ef4cdd23f4007ab84356dddae558b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "### Self Attention demo"
      ],
      "metadata": {
        "id": "SxU7pQb-IOIa"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JoIT5B4qJynv",
        "outputId": "1faebd80-d466-4206-bbe2-ab4e3af28f7a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Output after self-attention:\n",
            "tensor([[-0.7295,  1.0207,  1.1616],\n",
            "        [-0.7925,  1.1056,  1.2716],\n",
            "        [-0.8489,  1.1816,  1.3702]])\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "import torch.nn.functional as F\n",
        "\n",
        "# Example input sequence\n",
        "input_sequence = torch.tensor([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]])\n",
        "\n",
        "# Generate random weights for Key, Query, and Value matrices\n",
        "random_weights_key = torch.randn(input_sequence.size(-1), input_sequence.size(-1))\n",
        "random_weights_query = torch.randn(input_sequence.size(-1), input_sequence.size(-1))\n",
        "random_weights_value = torch.randn(input_sequence.size(-1), input_sequence.size(-1))\n",
        "\n",
        "# Compute Key, Query, and Value matrices\n",
        "key = torch.matmul(input_sequence, random_weights_key)\n",
        "query = torch.matmul(input_sequence, random_weights_query)\n",
        "value = torch.matmul(input_sequence, random_weights_value)\n",
        "\n",
        "# Compute attention scores\n",
        "attention_scores = torch.matmul(query, key.T) / torch.sqrt(torch.tensor(query.size(-1), dtype=torch.float32))\n",
        "\n",
        "# Apply softmax to obtain attention weights\n",
        "attention_weights = F.softmax(attention_scores, dim=-1)\n",
        "\n",
        "# Compute weighted sum of Value vectors\n",
        "output = torch.matmul(attention_weights, value)\n",
        "\n",
        "print(\"Output after self-attention:\")\n",
        "print(output)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Positional Encoding Implementation"
      ],
      "metadata": {
        "id": "Vim6tDefIVKa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "\n",
        "class PositionalEncoding(nn.Module):\n",
        "    def __init__(self, d_model, max_len=5000):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "\n",
        "        # Compute positional encodings\n",
        "        pe = torch.zeros(max_len, d_model)\n",
        "        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n",
        "        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-torch.log(\n",
        "        torch.tensor(10000.0)) / d_model))\n",
        "        pe[:, 0::2] = torch.sin(position * div_term)\n",
        "        pe[:, 1::2] = torch.cos(position * div_term)\n",
        "        pe = pe.unsqueeze(0)\n",
        "        self.register_buffer('pe', pe)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = x + x + self.pe[:, :x.size(1)]\n",
        "        return x\n",
        "\n",
        "# Example usage\n",
        "d_model = 512\n",
        "max_len = 100\n",
        "num_heads = 8\n",
        "\n",
        "# Positional encoding\n",
        "pos_encoder = PositionalEncoding(d_model, max_len)\n",
        "\n",
        "# Example input sequence\n",
        "input_sequence = torch.randn(5, max_len, d_model)\n",
        "\n",
        "# Apply positional encoding\n",
        "input_sequence = pos_encoder(input_sequence)\n",
        "print(\"Positional Encoding of input sequence:\")\n",
        "print(input_sequence.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7MjySsYY8Ut1",
        "outputId": "3a1bc23e-e28e-49b7-a68f-74ae80e48563"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Positional Encoding of input sequence:\n",
            "torch.Size([5, 100, 512])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Muti-Head Attention Implementation"
      ],
      "metadata": {
        "id": "y4L5o77fIdMR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Code implementation of Multi-Head Attention\n",
        "import math\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "\n",
        "class MultiHeadAttention(nn.Module):\n",
        "    def __init__(self, d_model, num_heads):\n",
        "        super(MultiHeadAttention, self).__init__()\n",
        "        self.num_heads = num_heads\n",
        "        self.d_model = d_model\n",
        "        assert d_model % num_heads == 0\n",
        "        self.depth = d_model // num_heads\n",
        "\n",
        "        # Linear projections for query, key, and value\n",
        "        self.query_linear = nn.Linear(d_model, d_model)\n",
        "        self.key_linear = nn.Linear(d_model, d_model)\n",
        "        self.value_linear = nn.Linear(d_model, d_model)\n",
        "\n",
        "        # Output linear projection\n",
        "        self.output_linear = nn.Linear(d_model, d_model)\n",
        "\n",
        "    def split_heads(self, x):\n",
        "      batch_size, seq_length, d_model = x.size()\n",
        "      return x.view(batch_size, seq_length, self.num_heads, self.depth).transpose(1, 2)\n",
        "\n",
        "    def forward(self, query, key, value, mask=None):\n",
        "\n",
        "        # Linear projections\n",
        "        query = self.query_linear(query)\n",
        "        key = self.key_linear(key)\n",
        "        value = self.value_linear(value)\n",
        "\n",
        "        # Split heads\n",
        "        query = self.split_heads(query)\n",
        "        key = self.split_heads(key)\n",
        "        value = self.split_heads(value)\n",
        "\n",
        "        # Scaled dot-product attention\n",
        "        scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(self.depth)\n",
        "\n",
        "        # Apply mask if provided\n",
        "        if mask is not None:\n",
        "            scores += scores.masked_fill(mask == 0, -1e9)\n",
        "\n",
        "        # Compute attention weights and apply softmax\n",
        "        attention_weights = torch.softmax(scores, dim=-1)\n",
        "\n",
        "        # Apply attention to values\n",
        "        attention_output = torch.matmul(attention_weights, value)\n",
        "\n",
        "        # Merge heads\n",
        "\n",
        "        batch_size, _, seq_length, d_k = attention_output.size()\n",
        "        attention_output = attention_output.transpose(1, 2).contiguous().view(batch_size, seq_length, self.d_model)\n",
        "\n",
        "        # Linear projection\n",
        "        attention_output = self.output_linear(attention_output)\n",
        "\n",
        "        return attention_output\n",
        "\n",
        "# Example usage\n",
        "d_model = 512\n",
        "max_len = 100\n",
        "num_heads = 8\n",
        "d_ff = 2048\n",
        "\n",
        "\n",
        "# Multi-head attention\n",
        "multihead_attn = MultiHeadAttention(d_model, num_heads)\n",
        "\n",
        "# Example input sequence\n",
        "input_sequence = torch.randn(5, max_len, d_model)\n",
        "\n",
        "# Multi-head attention\n",
        "attention_output= multihead_attn(input_sequence, input_sequence, input_sequence)\n",
        "print(\"attention_output shape:\", attention_output.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8_rdfpAJ5kPk",
        "outputId": "a856afba-4d09-4027-9dac-c7b10b5763c6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "attention_output shape: torch.Size([5, 100, 512])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Feed Forward Implementation"
      ],
      "metadata": {
        "id": "pONSFeoYIi3A"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class FeedForward(nn.Module):\n",
        "    def __init__(self, d_model, d_ff):\n",
        "        super(FeedForward, self).__init__()\n",
        "        self.linear1 = nn.Linear(d_model, d_ff)\n",
        "        self.linear2 = nn.Linear(d_ff, d_model)\n",
        "        self.relu = nn.ReLU()\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.relu(self.linear1(x))\n",
        "        x = self.linear2(x)\n",
        "        return x\n",
        "\n",
        "# Example usage\n",
        "d_model = 512\n",
        "max_len = 100\n",
        "num_heads = 8\n",
        "d_ff = 2048\n",
        "\n",
        "\n",
        "\n",
        "# Multi-head attention\n",
        "multihead_attn = MultiHeadAttention(d_model, num_heads)\n",
        "\n",
        "# Feed-forward network\n",
        "ff_network = FeedForward(d_model, d_ff)\n",
        "\n",
        "# Example input sequence\n",
        "input_sequence = torch.randn(5, max_len, d_model)\n",
        "\n",
        "# Multi-head attention\n",
        "attention_output= multihead_attn(input_sequence, input_sequence, input_sequence)\n",
        "\n",
        "# Feed-forward network\n",
        "output_ff = ff_network(attention_output)\n",
        "print('input_sequence',input_sequence.shape)\n",
        "print(\"output_ff\", output_ff.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yYdnz2xPzAJr",
        "outputId": "de5c4033-d53e-4b16-b75a-faf451e5ada6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "input_sequence torch.Size([5, 100, 512])\n",
            "output_ff torch.Size([5, 100, 512])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Encoder Implementation"
      ],
      "metadata": {
        "id": "06gAAPHLIqAG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch.nn as nn\n",
        "\n",
        "class EncoderLayer(nn.Module):\n",
        "    def __init__(self, d_model, num_heads, d_ff, dropout):\n",
        "        super(EncoderLayer, self).__init__()\n",
        "        self.self_attention = MultiHeadAttention(d_model, num_heads)\n",
        "        self.feed_forward = FeedForward(d_model, d_ff)\n",
        "        self.norm1 = nn.LayerNorm(d_model)\n",
        "        self.norm2 = nn.LayerNorm(d_model)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, x, mask):\n",
        "\n",
        "        # Self-attention layer\n",
        "        attention_output= self.self_attention(x, x,\n",
        "        x, mask)\n",
        "        attention_output = self.dropout(attention_output)\n",
        "        x = x + attention_output\n",
        "        x = self.norm1(x)\n",
        "\n",
        "        # Feed-forward layer\n",
        "        feed_forward_output = self.feed_forward(x)\n",
        "        feed_forward_output = self.dropout(feed_forward_output)\n",
        "        x = x + feed_forward_output\n",
        "        x = self.norm2(x)\n",
        "\n",
        "        return x\n",
        "\n",
        "d_model = 512\n",
        "max_len = 100\n",
        "num_heads = 8\n",
        "d_ff = 2048\n",
        "\n",
        "\n",
        "# Multi-head attention\n",
        "encoder_layer = EncoderLayer(d_model, num_heads, d_ff, 0.1)\n",
        "\n",
        "# Example input sequence\n",
        "input_sequence = torch.randn(1, max_len, d_model)\n",
        "\n",
        "# Multi-head attention\n",
        "encoder_output= encoder_layer(input_sequence, None)\n",
        "print(\"encoder output shape:\", encoder_output.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kURJdc_y8o3O",
        "outputId": "dcbb2f3d-37a3-48f1-f6d9-536d74717204"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "encoder output shape: torch.Size([1, 100, 512])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Decoder Implementation"
      ],
      "metadata": {
        "id": "7gaNWfhaIuvD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch.nn as nn\n",
        "\n",
        "class DecoderLayer(nn.Module):\n",
        "    def __init__(self, d_model, num_heads, d_ff, dropout):\n",
        "        super(DecoderLayer, self).__init__()\n",
        "        self.masked_self_attention = MultiHeadAttention(d_model, num_heads)\n",
        "        self.enc_dec_attention = MultiHeadAttention(d_model, num_heads)\n",
        "        self.feed_forward = FeedForward(d_model, d_ff)\n",
        "        self.norm1 = nn.LayerNorm(d_model)\n",
        "        self.norm2 = nn.LayerNorm(d_model)\n",
        "        self.norm3 = nn.LayerNorm(d_model)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, x, encoder_output, src_mask, tgt_mask):\n",
        "\n",
        "        # Masked self-attention layer\n",
        "        self_attention_output= self.masked_self_attention(x, x, x, tgt_mask)\n",
        "        self_attention_output = self.dropout(self_attention_output)\n",
        "        x = x + self_attention_output\n",
        "        x = self.norm1(x)\n",
        "\n",
        "        # Encoder-decoder attention layer\n",
        "        enc_dec_attention_output= self.enc_dec_attention(x, encoder_output,\n",
        "        encoder_output, src_mask)\n",
        "        enc_dec_attention_output = self.dropout(enc_dec_attention_output)\n",
        "        x = x + enc_dec_attention_output\n",
        "        x = self.norm2(x)\n",
        "\n",
        "        # Feed-forward layer\n",
        "        feed_forward_output = self.feed_forward(x)\n",
        "        feed_forward_output = self.dropout(feed_forward_output)\n",
        "        x = x + feed_forward_output\n",
        "        x = self.norm3(x)\n",
        "\n",
        "        return x\n",
        "\n",
        "# Define the DecoderLayer parameters\n",
        "d_model = 512  # Dimensionality of the model\n",
        "num_heads = 8  # Number of attention heads\n",
        "d_ff = 2048    # Dimensionality of the feed-forward network\n",
        "dropout = 0.1  # Dropout probability\n",
        "batch_size = 1 # Batch Size\n",
        "max_len = 100  # Max length of Sequence\n",
        "\n",
        "# Define the DecoderLayer instance\n",
        "decoder_layer = DecoderLayer(d_model, num_heads, d_ff, dropout)\n",
        "\n",
        "\n",
        "src_mask = torch.rand(batch_size, max_len, max_len) > 0.5\n",
        "tgt_mask = torch.tril(torch.ones(max_len, max_len)).unsqueeze(0) == 0\n",
        "\n",
        "# Pass the input tensors through the DecoderLayer\n",
        "output = decoder_layer(input_sequence, encoder_output, src_mask, tgt_mask)\n",
        "\n",
        "# Output shape\n",
        "print(\"Output shape:\", output.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DhhltzcxAS3m",
        "outputId": "cf31d5c9-513f-44aa-d7f3-a733ced40393"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Output shape: torch.Size([1, 100, 512])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Transformer Implementation"
      ],
      "metadata": {
        "id": "Zj4DGGLHIzP2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class Transformer(nn.Module):\n",
        "    def __init__(self, src_vocab_size, tgt_vocab_size, d_model, num_heads, num_layers, d_ff, max_len, dropout):\n",
        "        super(Transformer, self).__init__()\n",
        "\n",
        "        self.encoder_embedding = nn.Embedding(src_vocab_size, d_model)\n",
        "        self.decoder_embedding = nn.Embedding(tgt_vocab_size, d_model)\n",
        "\n",
        "        self.positional_encoding = PositionalEncoding(d_model, max_len)\n",
        "\n",
        "        self.encoder_layers = nn.ModuleList([EncoderLayer(d_model, num_heads, d_ff, dropout) for _ in range(num_layers)])\n",
        "        self.decoder_layers = nn.ModuleList([DecoderLayer(d_model, num_heads, d_ff, dropout) for _ in range(num_layers)])\n",
        "\n",
        "        self.linear = nn.Linear(d_model, tgt_vocab_size)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def generate_mask(self, src, tgt):\n",
        "        src_mask = (src != 0).unsqueeze(1).unsqueeze(2)\n",
        "        tgt_mask = (tgt != 0).unsqueeze(1).unsqueeze(3)\n",
        "        seq_length = tgt.size(1)\n",
        "        nopeak_mask = (1 - torch.triu(torch.ones(1, seq_length, seq_length), diagonal=1)).bool()\n",
        "        tgt_mask = tgt_mask & nopeak_mask\n",
        "        return src_mask, tgt_mask\n",
        "\n",
        "    def forward(self, src, tgt):\n",
        "        src_mask, tgt_mask = self.generate_mask(src, tgt)\n",
        "\n",
        "        encoder_embedding = self.encoder_embedding(src)\n",
        "        en_positional_encoding = self.positional_encoding(encoder_embedding)\n",
        "        src_embedded = self.dropout(en_positional_encoding)\n",
        "\n",
        "        decoder_embedding = self.decoder_embedding(tgt)\n",
        "        de_positional_encoding = self.positional_encoding(decoder_embedding)\n",
        "        tgt_embedded = self.dropout(de_positional_encoding)\n",
        "\n",
        "        enc_output = src_embedded\n",
        "        for enc_layer in self.encoder_layers:\n",
        "            enc_output = enc_layer(enc_output, src_mask)\n",
        "\n",
        "        dec_output = tgt_embedded\n",
        "        for dec_layer in self.decoder_layers:\n",
        "            dec_output = dec_layer(dec_output, enc_output, src_mask, tgt_mask)\n",
        "\n",
        "        output = self.linear(dec_output)\n",
        "        return output\n",
        "\n",
        "src_vocab_size = 5000\n",
        "tgt_vocab_size = 5000\n",
        "d_model = 512\n",
        "num_heads = 8\n",
        "num_layers = 6\n",
        "d_ff = 2048\n",
        "max_len = 100\n",
        "dropout = 0.1\n",
        "\n",
        "transformer = Transformer(src_vocab_size, tgt_vocab_size, d_model, num_heads, num_layers, d_ff, max_len, dropout)\n",
        "\n",
        "# Generate random sample data\n",
        "src_data = torch.randint(1, src_vocab_size, (5, max_len))  # (batch_size, seq_length)\n",
        "tgt_data = torch.randint(1, tgt_vocab_size, (5, max_len))  # (batch_size, seq_length)\n",
        "print(\"Transformer model output shape:\",transformer(src_data, tgt_data[:, :-1]).shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t8x3AKI7tUFN",
        "outputId": "da8c4624-47d1-4510-ce81-4cb2e3a310d3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Transformer model output shape: torch.Size([5, 99, 5000])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Training and Evaluation of Transformer Model"
      ],
      "metadata": {
        "id": "ES86Q9u-I67W"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "\n",
        "criterion = nn.CrossEntropyLoss(ignore_index=0)\n",
        "optimizer = optim.Adam(transformer.parameters(), lr=0.0001, betas=(0.9, 0.98), eps=1e-9)\n",
        "\n",
        "# Training loop\n",
        "transformer.train()\n",
        "\n",
        "for epoch in range(10):\n",
        "    optimizer.zero_grad()\n",
        "    output = transformer(src_data, tgt_data[:, :-1])\n",
        "    loss = criterion(output.contiguous().view(-1, tgt_vocab_size), tgt_data[:, 1:].contiguous().view(-1))\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    print(f\"Epoch {epoch+1}: Loss= {loss.item():.4f}\")\n",
        "\n",
        "\n",
        "#Dummy Data\n",
        "src_data = torch.randint(1, src_vocab_size, (5, max_len))  # (batch_size, seq_length)\n",
        "tgt_data = torch.randint(1, tgt_vocab_size, (5, max_len))  # (batch_size, seq_length)\n",
        "\n",
        "# Evaluation loop\n",
        "transformer.eval()\n",
        "with torch.no_grad():\n",
        "    output = transformer(src_data, tgt_data[:, :-1])\n",
        "    loss = criterion(output.contiguous().view(-1, tgt_vocab_size), tgt_data[:, 1:].contiguous().view(-1))\n",
        "    print(f\"\\nEvaluation Loss for dummy data= {loss.item():.4f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VuZAZvdW7gio",
        "outputId": "4e2e5a46-444a-4218-c5d8-31bed1e6383a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1: Loss= 8.6726\n",
            "Epoch 2: Loss= 8.1731\n",
            "Epoch 3: Loss= 7.8844\n",
            "Epoch 4: Loss= 7.6743\n",
            "Epoch 5: Loss= 7.4506\n",
            "Epoch 6: Loss= 7.1867\n",
            "Epoch 7: Loss= 6.9412\n",
            "Epoch 8: Loss= 6.6558\n",
            "Epoch 9: Loss= 6.4241\n",
            "Epoch 10: Loss= 6.1700\n",
            "\n",
            "Evaluation Loss for dummy data= 8.7536\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## BERT Implementation"
      ],
      "metadata": {
        "id": "b7_fim1pJCab"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import BertModel, BertTokenizer\n",
        "\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "model = BertModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "inputs = tokenizer(\"Hello, world!\", return_tensors=\"pt\")\n",
        "outputs = model(**inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281,
          "referenced_widgets": [
            "33c934ff057c4ee882294a4d8b5d08a2",
            "322c6638cacc429ca0ec6356df2d16d8",
            "55b6898f66c343f6bbc951c8034c6a14",
            "8aabd47a2b884fecb183f82c87c37c2f",
            "f1967b248be646978d25b37b19c189d0",
            "a78558c8f6c24890b2da8ef4ffba7fbc",
            "45fddca2236c4f399d5a6f0a24b7ffde",
            "7d7c703f325b4c479e45ff367774468f",
            "407c44f6938349a1b2da55279656b7b4",
            "4e2090ce05ed4fb49e09b8ceb42dce09",
            "7a8dc70a43264e89bf6611832e1e0493",
            "ceaaaf35614b4df4afd9a8b6b5964cae",
            "63296e04a09c4326888a880af0b47f45",
            "ce0f2d10ca3a4b9d89eb68a08356c61a",
            "e06b1ac6facb464fa093c1049ee2b986",
            "50faee8731874d4783c01f8fd2afcc61",
            "5d54676ba2784cb4a9b543cae8a23780",
            "1949d7f5bafd48cb8ef27c4e0901b227",
            "3716b098d44e4515b092ee90f39dc735",
            "6ce011ede20c4bbe9df3edcf2aeddbb1",
            "782faed5bbda4757b5875ed3592a8465",
            "93fc262be1184457b0820cc4e4bbe388",
            "a91a61753a8c4555812a635e7e8a6ed5",
            "a8c18751bbe94cc8ab5264c30ffbd01c",
            "74fa3150483c48da9a63b06602250f88",
            "481967cb0649479183832a6506af35b8",
            "b7d505edf2744176b8adaecebb512194",
            "ab2759518b90459181221d142233ffcd",
            "f6a7e926af1b44b79a5840c39e1ec3b4",
            "84c3b9adc4e848c3851e9b866ad90bd7",
            "ec47a105805947fb88196be17a39a7c3",
            "40170304fe1a46d9930fd1e4913e0efd",
            "0f884d80f1204955b185a89a71b248a2",
            "34b88f3f0ac7409a9a28a0ca3640b654",
            "06112d4e32034dc3a1de278df3d3c488",
            "ee5bfc02c18342a3a1c4718871a979d8",
            "e86c4cbe362742f280e3b25b16e936e0",
            "a6f017546b64402e9ad363773d9e1c10",
            "b8ccc02294f745f8aad3ebe5f7e5622a",
            "29abaad8458e40309e1326176beb12f5",
            "13013f89afdd4a8195ca2a26433cfbda",
            "0487c1da1767418292ad5dccb300c1de",
            "febfc7ccfd3544fba5d25d6ccc691c04",
            "731d7eee9f1a4099a47d4fdc3ea7048b",
            "c4d2e924d1344186ba7add7b2facaf95",
            "ea0f792474f24efe833dd68b2cf35f72",
            "e4ca83b501544c3cb42daeee40255b26",
            "f8d71f1116da42e2b02e458454665663",
            "8697f42fdfed49a0935080467402e738",
            "0a1c9ddc24334e1bb6fd7ac915b45978",
            "2042a701e05440faad772ee7e688f247",
            "39d938eef56e438ba5ba6ad544f40f66",
            "ce2c1b320ce743459d49701891fc5800",
            "62bc1930a4df4a8eb9ff5283478a29ee",
            "ab4ef4cdd23f4007ab84356dddae558b"
          ]
        },
        "id": "1BFeChw6_T03",
        "outputId": "20397c8a-cd5c-40a4-ba9c-b9c2429755bf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/48.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "33c934ff057c4ee882294a4d8b5d08a2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ceaaaf35614b4df4afd9a8b6b5964cae"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a91a61753a8c4555812a635e7e8a6ed5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "34b88f3f0ac7409a9a28a0ca3640b654"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/440M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c4d2e924d1344186ba7add7b2facaf95"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "outputs"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pz5eOYk2_WnD",
        "outputId": "1604a003-edd3-487e-cfe0-7235b8d910ab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BaseModelOutputWithPoolingAndCrossAttentions(last_hidden_state=tensor([[[-0.0781,  0.1587,  0.0400,  ..., -0.2805,  0.0248,  0.4081],\n",
              "         [-0.2016,  0.1781,  0.4184,  ..., -0.2522,  0.3630, -0.0979],\n",
              "         [-0.7156,  0.6751,  0.6017,  ..., -1.1032,  0.0797,  0.0567],\n",
              "         [ 0.0527, -0.1483,  1.3609,  ..., -0.4513,  0.1274,  0.2655],\n",
              "         [-0.7122, -0.4815, -0.1438,  ...,  0.5602, -0.1062, -0.1301],\n",
              "         [ 0.9955,  0.1328, -0.0621,  ...,  0.2460, -0.6502, -0.3296]]],\n",
              "       grad_fn=<NativeLayerNormBackward0>), pooler_output=tensor([[-0.8130, -0.2470, -0.7289,  0.5582,  0.3357, -0.0758,  0.7851,  0.1526,\n",
              "         -0.5705, -0.9997, -0.3183,  0.7643,  0.9550,  0.5801,  0.9046, -0.6037,\n",
              "         -0.3113, -0.5445,  0.3740, -0.4197,  0.5471,  0.9996,  0.0560,  0.2710,\n",
              "          0.3869,  0.9316, -0.7260,  0.8900,  0.9311,  0.5901, -0.5208,  0.0532,\n",
              "         -0.9711, -0.1791, -0.8414, -0.9663,  0.2318, -0.6239,  0.0885,  0.1203,\n",
              "         -0.8333,  0.1662,  0.9993,  0.1384,  0.1207, -0.3476, -1.0000,  0.2947,\n",
              "         -0.7443,  0.7037,  0.6978,  0.5853,  0.0875,  0.4013,  0.3722,  0.1009,\n",
              "         -0.1470,  0.1421, -0.2055, -0.4406, -0.6010,  0.2476, -0.7887, -0.8612,\n",
              "          0.8639,  0.7504, -0.0738, -0.2541,  0.0941, -0.1272,  0.7828,  0.1683,\n",
              "          0.0685, -0.8279,  0.4741,  0.2687, -0.6123,  1.0000, -0.3837, -0.9341,\n",
              "          0.5166,  0.5990,  0.5714, -0.2885,  0.4897, -1.0000,  0.2800, -0.1625,\n",
              "         -0.9728,  0.2292,  0.3729, -0.1447,  0.2490,  0.5224, -0.5050, -0.3634,\n",
              "         -0.2048, -0.7688, -0.2677, -0.1745, -0.0355, -0.2574, -0.1838, -0.3517,\n",
              "          0.2785, -0.3823, -0.3204,  0.4208, -0.0671,  0.6005,  0.3758, -0.3386,\n",
              "          0.4421, -0.9251,  0.5425, -0.2365, -0.9684, -0.5510, -0.9714,  0.4726,\n",
              "         -0.2355, -0.3178,  0.8958,  0.1285,  0.2222,  0.0103, -0.5784, -1.0000,\n",
              "         -0.5691, -0.5153, -0.0901, -0.1982, -0.9424, -0.9055,  0.4781,  0.9141,\n",
              "          0.0904,  0.9976, -0.2006,  0.8990, -0.3713, -0.6045,  0.5630, -0.3681,\n",
              "          0.7174,  0.1177, -0.4574,  0.1722, -0.0565,  0.2068, -0.5352, -0.1658,\n",
              "         -0.6867, -0.8909, -0.3266,  0.9221, -0.3681, -0.8595, -0.1558, -0.1426,\n",
              "         -0.4466,  0.7497,  0.5588,  0.3189, -0.2639,  0.3429,  0.1234,  0.4654,\n",
              "         -0.6938,  0.0370,  0.2499, -0.3322, -0.6972, -0.9597, -0.3168,  0.3913,\n",
              "          0.9778,  0.6148,  0.2344,  0.5841, -0.1060,  0.6453, -0.9023,  0.9467,\n",
              "         -0.2828,  0.2772, -0.2568,  0.3116, -0.7656,  0.2149,  0.7538, -0.4707,\n",
              "         -0.7183,  0.0423, -0.3983, -0.2417, -0.6220,  0.3196, -0.2374, -0.2951,\n",
              "          0.0163,  0.8490,  0.9524,  0.6414,  0.1112,  0.5715, -0.7746, -0.4595,\n",
              "         -0.0131,  0.1861,  0.2833,  0.9822, -0.5268, -0.0747, -0.8972, -0.9714,\n",
              "         -0.1566, -0.8502, -0.1051, -0.6196,  0.5066,  0.1431,  0.4483,  0.3532,\n",
              "         -0.9523, -0.6036,  0.2748, -0.2628,  0.3665, -0.1634,  0.2737,  0.8655,\n",
              "         -0.3408,  0.4985,  0.8835, -0.7243, -0.7103,  0.7471, -0.1439,  0.7210,\n",
              "         -0.4727,  0.9464,  0.7754,  0.6428, -0.8258, -0.5042, -0.7309, -0.5623,\n",
              "          0.0212,  0.0334,  0.8216,  0.5628,  0.2801,  0.3605, -0.5478,  0.9828,\n",
              "         -0.3884, -0.9035, -0.2619, -0.0373, -0.9656,  0.6856,  0.3733,  0.0614,\n",
              "         -0.3911, -0.5584, -0.9042,  0.8116,  0.0722,  0.9598,  0.0835, -0.8356,\n",
              "         -0.5096, -0.8548, -0.2696, -0.1833, -0.3283, -0.1286, -0.9169,  0.3966,\n",
              "          0.3708,  0.4118, -0.5972,  0.9906,  1.0000,  0.9055,  0.8319,  0.7821,\n",
              "         -0.9966, -0.4511,  0.9999, -0.9403, -1.0000, -0.8683, -0.5315,  0.2191,\n",
              "         -1.0000, -0.2363,  0.1403, -0.8282,  0.4823,  0.9450,  0.9512, -1.0000,\n",
              "          0.5255,  0.8806, -0.6060,  0.8864, -0.2879,  0.9407,  0.5752,  0.0417,\n",
              "         -0.1602,  0.2897, -0.8608, -0.7736, -0.3691, -0.5601,  0.9836,  0.1345,\n",
              "         -0.7325, -0.8195,  0.0158, -0.1550, -0.3395, -0.9400, -0.0907,  0.4092,\n",
              "          0.6562,  0.0334,  0.2541, -0.5023,  0.2764,  0.0445,  0.0117,  0.6056,\n",
              "         -0.8882, -0.4208, -0.4594, -0.2406, -0.4967, -0.9285,  0.9085, -0.4580,\n",
              "          0.7120,  1.0000,  0.3074, -0.7186,  0.4775,  0.1430, -0.2709,  1.0000,\n",
              "          0.7428, -0.9434, -0.4836,  0.4347, -0.3803, -0.4038,  0.9934, -0.2347,\n",
              "         -0.5286, -0.1938,  0.9469, -0.9670,  0.9769, -0.7885, -0.9335,  0.9128,\n",
              "          0.8891, -0.5778, -0.3787,  0.0850, -0.6286,  0.2903, -0.9003,  0.5381,\n",
              "          0.3541,  0.0589,  0.8031, -0.7100, -0.5081,  0.1781, -0.6506, -0.2105,\n",
              "          0.6789,  0.4671, -0.3476,  0.0115, -0.3682, -0.0806, -0.9431,  0.3343,\n",
              "          1.0000, -0.1909,  0.5555, -0.3358,  0.0263, -0.1470,  0.4349,  0.5208,\n",
              "         -0.2424, -0.7325,  0.7086, -0.9245, -0.9652,  0.7596,  0.1530, -0.1946,\n",
              "          0.9997,  0.4134,  0.1137,  0.2351,  0.9249, -0.0421,  0.4099,  0.8266,\n",
              "          0.9554, -0.1834,  0.5092,  0.6651, -0.7831, -0.2729, -0.5306, -0.0974,\n",
              "         -0.8989,  0.1261, -0.9079,  0.9355,  0.7817,  0.3302,  0.1569,  0.5768,\n",
              "          1.0000, -0.4562,  0.6240, -0.0049,  0.7729, -0.9976, -0.7388, -0.2276,\n",
              "          0.0201, -0.6561, -0.2721,  0.2316, -0.9417,  0.7118,  0.4029, -0.9698,\n",
              "         -0.9794, -0.2359,  0.7496,  0.0040, -0.9167, -0.5806, -0.4866,  0.4962,\n",
              "         -0.1977, -0.9085, -0.0888, -0.2179,  0.3301, -0.0793,  0.5415,  0.7117,\n",
              "          0.6331, -0.2443,  0.0208, -0.0152, -0.7002,  0.7866, -0.6876, -0.7593,\n",
              "         -0.1856,  1.0000, -0.5540,  0.7209,  0.6980,  0.5925, -0.0925,  0.1437,\n",
              "          0.8451,  0.2209, -0.6994, -0.7470, -0.5233, -0.2470,  0.5377,  0.0122,\n",
              "          0.6115,  0.6328,  0.5893,  0.0072,  0.0707, -0.1691,  0.9949, -0.0650,\n",
              "          0.0342, -0.3035,  0.0180, -0.2952, -0.2453,  1.0000,  0.2287,  0.3614,\n",
              "         -0.9741, -0.6818, -0.8467,  1.0000,  0.7774, -0.4980,  0.5307,  0.6857,\n",
              "         -0.0855,  0.7175, -0.1386, -0.3548,  0.2635,  0.0658,  0.8976, -0.5248,\n",
              "         -0.9286, -0.4457,  0.3075, -0.9184,  0.9985, -0.3725, -0.1925, -0.3857,\n",
              "          0.3529,  0.0612, -0.1111, -0.9667,  0.0296,  0.2127,  0.9023,  0.2146,\n",
              "         -0.5105, -0.8216,  0.6384,  0.5562, -0.7653, -0.9036,  0.9096, -0.9726,\n",
              "          0.6010,  1.0000,  0.3428, -0.2678, -0.0040, -0.2032,  0.1948, -0.2583,\n",
              "          0.6594, -0.8841, -0.3110, -0.1139,  0.2070, -0.0499,  0.0766,  0.6247,\n",
              "          0.1220, -0.4255, -0.4315,  0.0340,  0.3513,  0.7545, -0.2616, -0.1054,\n",
              "         -0.0310, -0.0160, -0.8557, -0.2032, -0.2896, -0.9998,  0.4847, -1.0000,\n",
              "          0.1944,  0.0238, -0.1356,  0.7747,  0.0479,  0.4497, -0.6311, -0.7206,\n",
              "          0.6178,  0.6761, -0.2757, -0.3905, -0.5928,  0.2279,  0.0707,  0.1233,\n",
              "         -0.5408,  0.7444, -0.1296,  1.0000,  0.0961, -0.6621, -0.9196,  0.0704,\n",
              "         -0.2162,  1.0000, -0.7794, -0.8807,  0.2784, -0.5381, -0.7780,  0.3047,\n",
              "         -0.0338, -0.6482, -0.7542,  0.8950,  0.8146, -0.5588,  0.3988, -0.2965,\n",
              "         -0.4299,  0.0082,  0.6164,  0.9624,  0.2798,  0.7427,  0.3798, -0.1202,\n",
              "          0.9364,  0.1309,  0.2478,  0.0943,  1.0000,  0.3100, -0.8320,  0.1281,\n",
              "         -0.9242, -0.1620, -0.8809,  0.1841,  0.2992,  0.8720, -0.2030,  0.8995,\n",
              "         -0.4650, -0.0161, -0.6945, -0.4291,  0.2841, -0.8518, -0.9624, -0.9653,\n",
              "          0.6546, -0.4158,  0.0047,  0.1431, -0.0799,  0.2424,  0.3504, -1.0000,\n",
              "          0.9038,  0.3259,  0.8399,  0.8938,  0.6032,  0.4055,  0.2465, -0.9604,\n",
              "         -0.9200, -0.1999, -0.2206,  0.5958,  0.5572,  0.8410,  0.2918, -0.5227,\n",
              "          0.0078, -0.3019, -0.4921, -0.9816,  0.4371, -0.5117, -0.8819,  0.9265,\n",
              "         -0.1755, -0.0597, -0.0823, -0.6530,  0.8747,  0.6324,  0.1266,  0.0842,\n",
              "          0.3756,  0.7381,  0.8790,  0.9607, -0.6325,  0.6952, -0.3407,  0.3745,\n",
              "          0.6012, -0.8744,  0.0708,  0.1280, -0.2589,  0.1242, -0.1201, -0.9274,\n",
              "          0.5940, -0.1563,  0.4428, -0.3287,  0.0695, -0.3981, -0.0400, -0.5623,\n",
              "         -0.6015,  0.5620,  0.2243,  0.8030,  0.7663, -0.0771, -0.4842, -0.1473,\n",
              "         -0.6306, -0.8541,  0.8141,  0.1151, -0.3174,  0.4548, -0.1950,  0.7887,\n",
              "          0.0674, -0.2464, -0.3487, -0.4228,  0.6947,  0.0686, -0.4925, -0.5855,\n",
              "          0.5164,  0.2094,  0.9996, -0.5488, -0.7718, -0.0737, -0.2518,  0.3538,\n",
              "         -0.3599, -1.0000,  0.3665, -0.2367,  0.6221, -0.5721,  0.3542, -0.5887,\n",
              "         -0.9486, -0.2115,  0.1483,  0.6009, -0.4153, -0.6647,  0.4821, -0.1477,\n",
              "          0.8825,  0.7133, -0.2224,  0.2536,  0.5956, -0.6499, -0.6185,  0.8514]],\n",
              "       grad_fn=<TanhBackward0>), hidden_states=None, past_key_values=None, attentions=None, cross_attentions=None)"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## GPT2 Implementation"
      ],
      "metadata": {
        "id": "xaf19WrAJG4U"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import GPT2LMHeadModel, GPT2Tokenizer\n",
        "\n",
        "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
        "model = GPT2LMHeadModel.from_pretrained('gpt2')\n",
        "\n",
        "input_text = \"Once upon a time, \"\n",
        "inputs=tokenizer(input_text,return_tensors='pt')\n",
        "output=tokenizer.decode(\n",
        "    model.generate(\n",
        "        **inputs,\n",
        "        max_new_tokens=100,\n",
        "      )[0],\n",
        "      skip_special_tokens=True\n",
        "  )\n",
        "input_ids = tokenizer(input_text, return_tensors='pt')\n",
        "\n",
        "output"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "id": "T58lT3uJ_baf",
        "outputId": "33278c68-f614-46ef-80f9-5833dd245d9c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Once upon a time, \\xa0I was a little bit of a fan of the original series, but I was also a little bit of a fan of the original series. I was a little bit of a fan of the original series, but I was also a little bit of a fan of the original series. I was a little bit of a fan of the original series, but I was also a little bit of a fan of the original series. I was a little bit of a fan of the original series, but I'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "R27HKtVE_4MQ"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}