BERT预训练脚本.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyMmiC8XAMZgguy/vYjD13Ai",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "332656d7d3014ff99c05d03abdfeda06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_207f8eb432a14d2f81a4dbf60a881485",
              "IPY_MODEL_b49f70a141a14209979f32e654e9fd68",
              "IPY_MODEL_078fc8dd03d64a428ee0056e36fa2fd7"
            ],
            "layout": "IPY_MODEL_2eb946003cc840169076bde37a8a5fdc"
          }
        },
        "207f8eb432a14d2f81a4dbf60a881485": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c8b3b6aa86934693b716f27cd389b573",
            "placeholder": "​",
            "style": "IPY_MODEL_eabbcdce8f344e34a60bf040c2b79576",
            "value": "Map: 100%"
          }
        },
        "b49f70a141a14209979f32e654e9fd68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b839f866ca3c4852a5ed28b18e6e5ef8",
            "max": 5391,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_44dfcb9742214aaa95e378ab5d7f19fe",
            "value": 5391
          }
        },
        "078fc8dd03d64a428ee0056e36fa2fd7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8e43f7f961f64a859f0742ed4f12b272",
            "placeholder": "​",
            "style": "IPY_MODEL_e6d4a5aded1b4b5db3b7b8ce09a600e4",
            "value": " 5391/5391 [00:15&lt;00:00, 352.72 examples/s]"
          }
        },
        "2eb946003cc840169076bde37a8a5fdc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c8b3b6aa86934693b716f27cd389b573": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eabbcdce8f344e34a60bf040c2b79576": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b839f866ca3c4852a5ed28b18e6e5ef8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "44dfcb9742214aaa95e378ab5d7f19fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8e43f7f961f64a859f0742ed4f12b272": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e6d4a5aded1b4b5db3b7b8ce09a600e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "01d242829aa545c8b14564ed8abe9579": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d8a4d824667f4a0c9180f00cd5f613c0",
              "IPY_MODEL_9ca041eb04354e9e9859d57353e6ae9e",
              "IPY_MODEL_806c784ff1544c6c98232d9229e7f09d"
            ],
            "layout": "IPY_MODEL_5290767cd2e0444c803c144c7e9ec201"
          }
        },
        "d8a4d824667f4a0c9180f00cd5f613c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1442e876943242f5aa7620002d2fe329",
            "placeholder": "​",
            "style": "IPY_MODEL_ada676e222fc41ad9b55bad35da7c300",
            "value": "Map: 100%"
          }
        },
        "9ca041eb04354e9e9859d57353e6ae9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8796592fa0484b5d9c040455e45e86ae",
            "max": 600,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bf432441fb9b4030a6ae0b4bfdcec18b",
            "value": 600
          }
        },
        "806c784ff1544c6c98232d9229e7f09d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0b5075d2aa5f496eb1f0c13bf0617c95",
            "placeholder": "​",
            "style": "IPY_MODEL_873fcc7f21ae4c5a85509836912ab6ac",
            "value": " 600/600 [00:01&lt;00:00, 442.19 examples/s]"
          }
        },
        "5290767cd2e0444c803c144c7e9ec201": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1442e876943242f5aa7620002d2fe329": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ada676e222fc41ad9b55bad35da7c300": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8796592fa0484b5d9c040455e45e86ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bf432441fb9b4030a6ae0b4bfdcec18b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0b5075d2aa5f496eb1f0c13bf0617c95": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "873fcc7f21ae4c5a85509836912ab6ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/DayDreamChaser/llm-todo/blob/main/BERT%E9%A2%84%E8%AE%AD%E7%BB%83%E8%84%9A%E6%9C%AC.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fX_S9Dm0sAoV",
        "outputId": "ac09a42c-eb87-411a-d83a-44c42d01c254"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (2.18.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets) (3.13.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.25.2)\n",
            "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (14.0.2)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
            "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.3.8)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.2)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets) (0.70.16)\n",
            "Requirement already satisfied: fsspec[http]<=2024.2.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.3)\n",
            "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.20.3)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->datasets) (4.10.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2024.2.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.4)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install datasets"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import concatenate_datasets, load_dataset\n",
        "\n",
        "wikitext2 = load_dataset(\"liuyanchen1015/VALUE_wikitext2_been_done\", split=\"train\")"
      ],
      "metadata": {
        "id": "oOdwAsWqs3OG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9d14f592-a29b-48e3-9044-2e10eb0c8820"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# wiki = load_dataset(\"wikipedia\", \"20220301.en\", split=\"train\")\n",
        "# # 仅保留 'text' 列\n",
        "# wiki = wiki.remove_columns([col for col in wiki.column_names if col != \"text\"])\n",
        "\n",
        "dataset = concatenate_datasets([wikitext2])\n",
        "# 将数据集合切分为 90% 用于训练，10% 用于测试\n",
        "d = dataset.train_test_split(test_size=0.1)"
      ],
      "metadata": {
        "id": "pa4SJKV7v6io"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "type(d[\"train\"])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 138
        },
        "id": "MGtIkkO4--8C",
        "outputId": "b8661fc4-e909-460a-8fbb-d7b7cdd0bd4d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "datasets.arrow_dataset.Dataset"
            ],
            "text/html": [
              "<div style=\"max-width:800px; border: 1px solid var(--colab-border-color);\"><style>\n",
              "      pre.function-repr-contents {\n",
              "        overflow-x: auto;\n",
              "        padding: 8px 12px;\n",
              "        max-height: 500px;\n",
              "      }\n",
              "\n",
              "      pre.function-repr-contents.function-repr-contents-collapsed {\n",
              "        cursor: pointer;\n",
              "        max-height: 100px;\n",
              "      }\n",
              "    </style>\n",
              "    <pre style=\"white-space: initial; background:\n",
              "         var(--colab-secondary-surface-color); padding: 8px 12px;\n",
              "         border-bottom: 1px solid var(--colab-border-color);\"><b>datasets.arrow_dataset.Dataset</b><br/>def __init__(arrow_table: Table, info: Optional[DatasetInfo]=None, split: Optional[NamedSplit]=None, indices_table: Optional[Table]=None, fingerprint: Optional[str]=None)</pre><pre class=\"function-repr-contents function-repr-contents-collapsed\" style=\"\"><a class=\"filepath\" style=\"display:none\" href=\"#\">/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py</a>A Dataset backed by an Arrow table.</pre>\n",
              "      <script>\n",
              "      if (google.colab.kernel.accessAllowed && google.colab.files && google.colab.files.view) {\n",
              "        for (const element of document.querySelectorAll('.filepath')) {\n",
              "          element.style.display = 'block'\n",
              "          element.onclick = (event) => {\n",
              "            event.preventDefault();\n",
              "            event.stopPropagation();\n",
              "            google.colab.files.view(element.textContent, 659);\n",
              "          };\n",
              "        }\n",
              "      }\n",
              "      for (const element of document.querySelectorAll('.function-repr-contents')) {\n",
              "        element.onclick = (event) => {\n",
              "          event.preventDefault();\n",
              "          event.stopPropagation();\n",
              "          element.classList.toggle('function-repr-contents-collapsed');\n",
              "        };\n",
              "      }\n",
              "      </script>\n",
              "      </div>"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(d[\"train\"]))\n",
        "print(len(d[\"test\"]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ltTVe2G7_K7x",
        "outputId": "46e979c9-8e32-4606-9be8-08f351934f3b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "5391\n",
            "600\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "d[\"train\"][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lBTVvF3c_Oig",
        "outputId": "6e8c7654-e90e-498e-d336-164a8e28fd4f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'sentence': 'Dawn revealed a large number of craters with low relief , indicating that they lie over a relatively soft surface , probably of water ice . One crater , with extremely low relief , is 270 km ( 170 mi ) in diameter , reminiscent of large , flat craters on Tethys and Iapetus . An unexpectedly large number of Cererian craters have central pits , and many have central peaks . Several bright spots done been observed by Dawn , the brightest spot ( \" Spot 5 \" ) located in the middle of an 80 @-@ kilometer ( 50 mi ) crater called Occator . From images taken of Ceres on 4 May 2015 , the secondary bright spot was revealed to actually be a group of scattered bright areas , possibly as many as ten . These bright features have an albedo of approximately 40 % that are caused by a substance on the surface , possibly ice or salts , reflecting sunlight . A haze periodically appears above Spot 5 , the best known bright spot , supporting the hypothesis that some sort of outgassing or sublimating ice formed the bright spots . In March 2016 , Dawn found definitive evidence of water molecules on the surface of Ceres at Oxo crater . JPL states : \" This water could be bound up in minerals or , alternatively , it could take the form of ice . \"',\n",
              " 'idx': 5096,\n",
              " 'score': 1}"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def dataset_to_text(dataset, output_filename=\"data.txt\", num=1000):\n",
        "  \"\"\"Utility function to save dataset text to disk,\n",
        "  useful for using the texts to train the tokenizer\n",
        "  (as the tokenizer accepts files)\"\"\"\n",
        "  cnt = 0\n",
        "  with open(output_filename, \"w\") as f:\n",
        "    for t in dataset[\"sentence\"]:\n",
        "      print(t, file=f)\n",
        "      # print(len(t), t)\n",
        "      cnt += 1\n",
        "      if cnt == num:\n",
        "        break\n",
        "\n",
        "    print(\"Write {} sample in {}\".format(num, output_filename))\n",
        "\n",
        "# save the training set to train.txt\n",
        "dataset_to_text(d[\"train\"], \"train.txt\", len(d[\"train\"]))\n",
        "# save the testing set to test.txt\n",
        "dataset_to_text(d[\"test\"], \"test.txt\", len(d[\"test\"]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KC-g3dvstJYg",
        "outputId": "dbd5f087-9914-4914-8017-1257fd051a5c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Write 5391 sample in train.txt\n",
            "Write 600 sample in test.txt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tokenizers import BertWordPieceTokenizer\n",
        "\n",
        "special_tokens = [\n",
        "\"[PAD]\", \"[UNK]\", \"[CLS]\", \"[SEP]\", \"[MASK]\", \"<S>\", \"<T>\"\n",
        "]\n",
        "# if you want to train the tokenizer on both sets\n",
        "# files = [\"train.txt\", \"test.txt\"]\n",
        "# training the tokenizer on the training set\n",
        "files = [\"train.txt\"]\n",
        "# 30,522 vocab is BERT's default vocab size, feel free to tweak\n",
        "vocab_size = 30_522\n",
        "# maximum sequence length, lowering will result to faster training (when increasing batch size)\n",
        "max_length = 512\n",
        "# whether to truncate\n",
        "# truncate_longer_samples = False\n",
        "truncate_longer_samples = True\n",
        "\n",
        "# initialize the WordPiece tokenizer\n",
        "tokenizer = BertWordPieceTokenizer()\n",
        "# train the tokenizer"
      ],
      "metadata": {
        "id": "sJdGjWlWwykq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "voFqfnsb1GAc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os, json\n",
        "from transformers import BertTokenizerFast\n",
        "\n",
        "tokenizer.train(files=files, vocab_size=vocab_size, special_tokens=special_tokens)\n",
        "# enable truncation up to the maximum 512 tokens\n",
        "tokenizer.enable_truncation(max_length=max_length)\n",
        "\n",
        "model_path = \"pretrained-bert\"\n",
        "# make the directory if not already there\n",
        "if not os.path.isdir(model_path):\n",
        "  os.mkdir(model_path)\n",
        "# save the tokenizer\n",
        "tokenizer.save_model(model_path)\n",
        "# dumping some of the tokenizer config to config file,\n",
        "# including special tokens, whether to lower case and the maximum sequence length\n",
        "with open(os.path.join(model_path, \"config.json\"), \"w\") as f:\n",
        "  tokenizer_cfg = {\n",
        "  \"do_lower_case\": True,\n",
        "  \"unk_token\": \"[UNK]\",\n",
        "  \"sep_token\": \"[SEP]\",\n",
        "  \"pad_token\": \"[PAD]\",\n",
        "  \"cls_token\": \"[CLS]\",\n",
        "  \"mask_token\": \"[MASK]\",\n",
        "  \"model_max_length\": max_length,\n",
        "  \"max_len\": max_length,\n",
        "  }\n",
        "  json.dump(tokenizer_cfg, f)\n",
        "\n",
        "# when the tokenizer is trained and configured, load it as BertTokenizerFast\n",
        "tokenizer = BertTokenizerFast.from_pretrained(model_path)"
      ],
      "metadata": {
        "id": "SxH1E28H0_PV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def encode_with_truncation(examples):\n",
        "  \"\"\"Mapping function to tokenize the sentences passed with truncation\"\"\"\n",
        "  return tokenizer(examples[\"sentence\"], truncation=True, padding=\"max_length\",\n",
        "  max_length=max_length, return_special_tokens_mask=True)\n",
        "\n",
        "def encode_without_truncation(examples):\n",
        "  \"\"\"Mapping function to tokenize the sentences passed without truncation\"\"\"\n",
        "  return tokenizer(examples[\"sentence\"], return_special_tokens_mask=True)\n",
        "\n",
        "# the encode function will depend on the truncate_longer_samples variable\n",
        "encode = encode_with_truncation if truncate_longer_samples else encode_without_truncation\n",
        "# tokenizing the train dataset\n",
        "train_dataset = d[\"train\"].map(encode, batched=True)\n",
        "# tokenizing the testing dataset\n",
        "test_dataset = d[\"test\"].map(encode, batched=True)\n",
        "if truncate_longer_samples:\n",
        "  # remove other columns and set input_ids and attention_mask as PyTorch tensors\n",
        "  train_dataset.set_format(type=\"torch\", columns=[\"input_ids\", \"attention_mask\"])\n",
        "  test_dataset.set_format(type=\"torch\", columns=[\"input_ids\", \"attention_mask\"])\n",
        "else:\n",
        "  # remove other columns, and remain them as Python lists\n",
        "  test_dataset.set_format(columns=[\"input_ids\", \"attention_mask\", \"special_tokens_mask\"])\n",
        "  train_dataset.set_format(columns=[\"input_ids\", \"attention_mask\", \"special_tokens_mask\"])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "332656d7d3014ff99c05d03abdfeda06",
            "207f8eb432a14d2f81a4dbf60a881485",
            "b49f70a141a14209979f32e654e9fd68",
            "078fc8dd03d64a428ee0056e36fa2fd7",
            "2eb946003cc840169076bde37a8a5fdc",
            "c8b3b6aa86934693b716f27cd389b573",
            "eabbcdce8f344e34a60bf040c2b79576",
            "b839f866ca3c4852a5ed28b18e6e5ef8",
            "44dfcb9742214aaa95e378ab5d7f19fe",
            "8e43f7f961f64a859f0742ed4f12b272",
            "e6d4a5aded1b4b5db3b7b8ce09a600e4",
            "01d242829aa545c8b14564ed8abe9579",
            "d8a4d824667f4a0c9180f00cd5f613c0",
            "9ca041eb04354e9e9859d57353e6ae9e",
            "806c784ff1544c6c98232d9229e7f09d",
            "5290767cd2e0444c803c144c7e9ec201",
            "1442e876943242f5aa7620002d2fe329",
            "ada676e222fc41ad9b55bad35da7c300",
            "8796592fa0484b5d9c040455e45e86ae",
            "bf432441fb9b4030a6ae0b4bfdcec18b",
            "0b5075d2aa5f496eb1f0c13bf0617c95",
            "873fcc7f21ae4c5a85509836912ab6ac"
          ]
        },
        "id": "gnLPQVaq1deZ",
        "outputId": "067792bd-dfa1-4131-e866-261221f12a31"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/5391 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "332656d7d3014ff99c05d03abdfeda06"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/600 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "01d242829aa545c8b14564ed8abe9579"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x = d[\"train\"][10]['sentence'].split(\".\")[0]\n",
        "x"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "id": "mUOs5a60Bwbt",
        "outputId": "ee99dc22-019c-4f59-ed6a-7a0d7f81781c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'In his foreword to John Marriott \\'s book , Thunderbirds Are Go ! , Anderson put forward several explanations for the series \\' enduring popularity : it \" contains elements that appeal to most children – danger , jeopardy and destruction '"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer(x, truncation=True, padding=\"max_length\", max_length=33, return_special_tokens_mask=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x0W2rmfhB0n7",
        "outputId": "3a4f1048-bebb-4682-bbe5-0d8a020523d0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [2, 502, 577, 23750, 649, 511, 1136, 29823, 13, 59, 1253, 18, 4497, 659, 762, 7, 18, 3704, 2041, 3048, 1064, 26668, 534, 492, 1005, 13, 12799, 4281, 32, 555, 8, 4265, 3], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'special_tokens_mask': [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]}"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y = tokenizer(x, truncation=False, return_special_tokens_mask=True)\n",
        "y"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MUn7BlZVC1Ly",
        "outputId": "9dc0dbec-5162-45ad-cedb-2df520a6daeb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [2, 502, 577, 23750, 649, 511, 1136, 29823, 13, 59, 1253, 18, 4497, 659, 762, 7, 18, 3704, 2041, 3048, 1064, 26668, 534, 492, 1005, 13, 12799, 4281, 32, 555, 8, 4265, 2603, 545, 3740, 511, 843, 1970, 178, 8021, 18, 19251, 362, 510, 6681, 3], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'special_tokens_mask': [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]}"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "help(tokenizer.decode)"
      ],
      "metadata": {
        "id": "CGiKDxLvBQ2p",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a4f40abd-5cae-49be-beaa-e77f1246f7c7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Help on method decode in module transformers.tokenization_utils_base:\n",
            "\n",
            "decode(token_ids: Union[int, List[int], ForwardRef('np.ndarray'), ForwardRef('torch.Tensor'), ForwardRef('tf.Tensor')], skip_special_tokens: bool = False, clean_up_tokenization_spaces: bool = None, **kwargs) -> str method of transformers.models.bert.tokenization_bert_fast.BertTokenizerFast instance\n",
            "    Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special\n",
            "    tokens and clean up tokenization spaces.\n",
            "    \n",
            "    Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`.\n",
            "    \n",
            "    Args:\n",
            "        token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):\n",
            "            List of tokenized input ids. Can be obtained using the `__call__` method.\n",
            "        skip_special_tokens (`bool`, *optional*, defaults to `False`):\n",
            "            Whether or not to remove special tokens in the decoding.\n",
            "        clean_up_tokenization_spaces (`bool`, *optional*):\n",
            "            Whether or not to clean up the tokenization spaces. If `None`, will default to\n",
            "            `self.clean_up_tokenization_spaces`.\n",
            "        kwargs (additional keyword arguments, *optional*):\n",
            "            Will be passed to the underlying model specific decode method.\n",
            "    \n",
            "    Returns:\n",
            "        `str`: The decoded sentence.\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.decode(y[\"input_ids\"])"
      ],
      "metadata": {
        "id": "sCP3ytci6cN4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "ad3a61cb-d2ea-4573-cb66-b6fa8eb6bcfe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'[CLS] in his foreword to john marriott\\'s book, thunderbirds are go!, anderson put forward several explanations for the series\\'enduring popularity : it \" contains elements that appeal to most children – danger, jeopardy and destruction [SEP]'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 288
        },
        "id": "WvoKTWydTnr4",
        "outputId": "baf53548-cc5f-4285-efd3-dc98b9aecb45"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "dict_keys(['input_ids', 'attention_mask'])\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "name 'chain' is not defined",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-16-cd2ced937203>\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mz\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_dataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mz\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mz\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mz\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-16-cd2ced937203>\u001b[0m in \u001b[0;36m<dictcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mz\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_dataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mz\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mz\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mz\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'chain' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "JMuVWZ1c2dnP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from itertools import chain\n",
        "# Main data processing function that will concatenate all texts from our dataset\n",
        "# and generate chunks of max_seq_length.\n",
        "\n",
        "def group_texts(examples):\n",
        "  # Concatenate all texts.\n",
        "  concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()}\n",
        "  total_length = len(concatenated_examples[list(examples.keys())[0]])\n",
        "  # We drop the small remainder, we could add padding if the model supported it instead of\n",
        "  # this drop, you can customize this part to your needs.\n",
        "  if total_length >= max_length:\n",
        "    total_length = (total_length // max_length) * max_length\n",
        "  # Split by chunks of max_len.\n",
        "  result = {\n",
        "  k: [t[i : i + max_length] for i in range(0, total_length, max_length)]\n",
        "  for k, t in concatenated_examples.items()\n",
        "  }\n",
        "  return result\n",
        "\n",
        "# Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws\n",
        "# away a remainder for each of those groups of 1,000 texts. You can adjust that batch_size here but\n",
        "# a higher value might be slower to preprocess.\n",
        "#\n",
        "# To speed up this part, we use multiprocessing. See the documentation of the map method\n",
        "#for more information:\n",
        "# https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map\n",
        "if not truncate_longer_samples:\n",
        "  train_dataset = train_dataset.map(group_texts, batched=True,\n",
        "  desc=f\"Grouping texts in chunks of {max_length}\")\n",
        "  test_dataset = test_dataset.map(group_texts, batched=True,\n",
        "  desc=f\"Grouping texts in chunks of {max_length}\")\n",
        "  # convert them from lists to torch tensors\n",
        "  train_dataset.set_format(\"torch\")\n",
        "  test_dataset.set_format(\"torch\")\n"
      ],
      "metadata": {
        "id": "D8x7cNxj2cmT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install accelerate -U"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K8NQCDvAafqk",
        "outputId": "b6b81acb-1279-4485-ac34-633d78f6d35a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (0.27.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate) (1.25.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (23.2)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate) (6.0.1)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.1.0+cu121)\n",
            "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.20.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.4.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (4.10.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.3)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2023.6.0)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.1.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (4.66.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.5)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2024.2.2)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "0oyToGNf2u9Z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import BertConfig, BertForMaskedLM\n",
        "from transformers import DataCollatorForLanguageModeling, TrainingArguments, Trainer\n",
        "\n",
        "# initialize the model with the config\n",
        "model_config = BertConfig(vocab_size=vocab_size, max_position_embeddings=max_length)\n",
        "model = BertForMaskedLM(config=model_config)\n",
        "# initialize the data collator, randomly masking 20% (default is 15%) of the tokens\n",
        "# for the Masked Language Modeling (MLM) task\n",
        "data_collator = DataCollatorForLanguageModeling(\n",
        "tokenizer=tokenizer, mlm=True, mlm_probability=0.2\n",
        ")\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "  output_dir=model_path, # output directory to where save model checkpoint\n",
        "  evaluation_strategy=\"steps\", # evaluate each `logging_steps` steps\n",
        "  overwrite_output_dir=True,\n",
        "  num_train_epochs=10, # number of training epochs, feel free to tweak\n",
        "  per_device_train_batch_size=10, # the training batch size, put it as high as your GPU memory fits\n",
        "  gradient_accumulation_steps=8, # accumulating the gradients before updating the weights\n",
        "  per_device_eval_batch_size=64, # evaluation batch size\n",
        "  logging_steps=1000, # evaluate, log and save model checkpoints every 1000 step\n",
        "  save_steps=1000,\n",
        "  # load_best_model_at_end=True, # whether to load the best model (in terms of loss)\n",
        "  # at the end of training\n",
        "  # save_total_limit=3, # whether you don't have much space so you\n",
        "  # let only 3 model weights saved in the disk\n",
        "  )\n",
        "\n",
        "trainer = Trainer(\n",
        "  model=model,\n",
        "  args=training_args,\n",
        "  data_collator=data_collator,\n",
        "  train_dataset=train_dataset,\n",
        "  eval_dataset=test_dataset,\n",
        "  )\n",
        "# train the model\n",
        "trainer.train()\n"
      ],
      "metadata": {
        "id": "X_6Q1VoU2vG3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# load the model checkpoint\n",
        "model = BertForMaskedLM.from_pretrained(os.path.join(model_path, \"checkpoint-10000\"))\n",
        "# load the tokenizer\n",
        "tokenizer = BertTokenizerFast.from_pretrained(model_path)\n",
        "fill_mask = pipeline(\"fill-mask\", model=model, tokenizer=tokenizer)\n",
        "# perform predictions\n",
        "examples = [\n",
        "\"Today's most trending hashtags on [MASK] is Donald Trump\",\n",
        "\"The [MASK] was cloudy yesterday, but today it's rainy.\",\n",
        "]\n",
        "for example in examples:\n",
        "  for prediction in fill_mask(example):\n",
        "    print(f\"{prediction['sequence']}, confidence: {prediction['score']}\")\n",
        "    print(\"=\"*50)\n"
      ],
      "metadata": {
        "id": "bX1fbmQd21ch"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}