demo.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rJ5ngOvLYZXd"
      },
      "source": [
        "# Setup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J0eME1NfZRr8",
        "outputId": "0cfb0478-26b2-4a0e-bfe7-84efae229546"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.26.1-py3-none-any.whl (6.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m89.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tensorboardX\n",
            "  Downloading tensorboardX-2.6-py2.py3-none-any.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 KB\u001b[0m \u001b[31m16.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: gdown in /usr/local/lib/python3.8/dist-packages (4.4.0)\n",
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.97-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m55.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting huggingface-hub<1.0,>=0.11.0\n",
            "  Downloading huggingface_hub-0.12.0-py3-none-any.whl (190 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m190.3/190.3 KB\u001b[0m \u001b[31m17.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m28.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (23.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from transformers) (2.25.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers) (3.9.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.8/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: protobuf<4,>=3.8.0 in /usr/local/lib/python3.8/dist-packages (from tensorboardX) (3.19.6)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.8/dist-packages (from gdown) (4.6.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.8/dist-packages (from gdown) (1.15.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.4.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (4.0.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (1.7.1)\n",
            "Installing collected packages: tokenizers, sentencepiece, tensorboardX, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.12.0 sentencepiece-0.1.97 tensorboardX-2.6 tokenizers-0.13.2 transformers-4.26.1\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers tensorboardX gdown sentencepiece"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "id": "AQRsiEZnO0LH",
        "outputId": "f169dd74-2522-4a2d-d02d-2069ca7ee3d5"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=10GjAH-Qgv5G2HLzftg2AybUyAHZCnPDF&confirm=t\n",
            "To: /content/trained_model.zip\n",
            "100%|██████████| 384M/384M [00:14<00:00, 25.8MB/s]\n"
          ]
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'trained_model.zip'"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import gdown\n",
        "\n",
        "url = \"https://drive.google.com/uc?id=10GjAH-Qgv5G2HLzftg2AybUyAHZCnPDF&confirm=t\"\n",
        "output = \"trained_model.zip\"\n",
        "gdown.download(url, output, quiet=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cf9oXpe3QDHp",
        "outputId": "cb572a52-49d6-47c4-88b3-15412755850a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Archive:  trained_model.zip\n",
            "  inflating: config.json             \n",
            "  inflating: pytorch_model.bin       \n"
          ]
        }
      ],
      "source": [
        "!unzip trained_model.zip\n",
        "!mkdir trained_model\n",
        "!mv config.json trained_model/config.json\n",
        "!mv pytorch_model.bin trained_model/pytorch_model.bin"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1e6qx0PdYjo3"
      },
      "source": [
        "# Inference implementation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "c5qx7yuVSDyr"
      },
      "outputs": [],
      "source": [
        "from itertools import chain\n",
        "\n",
        "import torch\n",
        "from transformers import T5Tokenizer\n",
        "from transformers.modeling_outputs import BaseModelOutput\n",
        "\n",
        "import definitions\n",
        "from model import T5WithSpan\n",
        "\n",
        "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "zKh4WsLBl9JP"
      },
      "outputs": [],
      "source": [
        "def init_tokenizer(backbone):\n",
        "  tokenizer = T5Tokenizer.from_pretrained(backbone)\n",
        "\n",
        "  special_tokens = []\n",
        "\n",
        "  # add domains\n",
        "  domains = definitions.ALL_DOMAINS + [\"general\"]\n",
        "  for domain in sorted(domains):\n",
        "    token = \"[\" + domain + \"]\"\n",
        "    special_tokens.append(token)\n",
        "\n",
        "  # add intents\n",
        "  intents = list(set(chain(*definitions.DIALOG_ACTS.values())))\n",
        "  for intent in sorted(intents):\n",
        "    token = \"[\" + intent + \"]\"\n",
        "    special_tokens.append(token)\n",
        "\n",
        "  # add slots\n",
        "  slots = list(set(definitions.ALL_INFSLOT + definitions.ALL_REQSLOT))\n",
        "\n",
        "  for slot in sorted(slots):\n",
        "    token = \"[value_\" + slot + \"]\"\n",
        "    special_tokens.append(token)\n",
        "\n",
        "  special_tokens.extend(definitions.SPECIAL_TOKENS)\n",
        "  tokenizer.add_special_tokens({\"additional_special_tokens\": special_tokens})\n",
        "\n",
        "  return tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 208,
          "referenced_widgets": [
            "634086bb039d47be9bfb5b6a8c5f1111",
            "33e590cba6724cf0b85bfd20dc70692a",
            "69863c5feb544ccbb8dac7587c85aea4",
            "38558746a61d46529c339cc5d36f07a7",
            "42b4d88e0a3045f6bb51784cac1102fd",
            "fad442dda7814ad0bda08c787cf902fd",
            "f47f4e66477f46198e01ae5a9f372e64",
            "fe9a01a140d74334b80e3bea098a9e59",
            "852da1a3fe014d01b052d110774342a2",
            "0db13575319c43edb7855a45c262c691",
            "61440360e85944d0910bc2a0a59eef3e",
            "a733126684fc4f74987b292ab4fb192a",
            "721f1a96192e4627b4846e20cc6346f7",
            "6d1219bf2ee64361885ebb8ac399bda7",
            "16cdf3e4d5414d15b24c016e47ba084d",
            "419ea03cd0b849f2b321144965745669",
            "0b446e715bc84ec89e0121ed03c76cb4",
            "0df7fb8a891e4f90a1b318e770081a5c",
            "84c44fd43f444dc4b65e21c284a45f8e",
            "92996c7d79664dfdbf588095fa9e620f",
            "e4dd40da3cf44fbb906e97adcd55f887",
            "ffd8432c713141eda314659ce8f6ef80"
          ]
        },
        "id": "pzXtZs5NSob_",
        "outputId": "e02ac476-686a-4e0d-cb4c-b286f81da7b7"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "634086bb039d47be9bfb5b6a8c5f1111",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)ve/main/spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a733126684fc4f74987b292ab4fb192a",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/1.21k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/transformers/models/t5/tokenization_t5.py:163: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5.\n",
            "For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`.\n",
            "- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding.\n",
            "- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding.\n",
            "- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value.\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "tokenizer = init_tokenizer(\"t5-small\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SvLvweJcOAyn",
        "outputId": "94fd90ff-f416-49a5-8c82-1a4f38084251"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at trained_model were not used when initializing T5WithSpan: ['span_head.weight', 'span_head.bias']\n",
            "- This IS expected if you are initializing T5WithSpan from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing T5WithSpan from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        }
      ],
      "source": [
        "# max_seq_len is set to a constant for the Gumbel-Softmax variant\n",
        "model = T5WithSpan.from_pretrained(\"trained_model\",\n",
        "                                    num_span=7,\n",
        "                                    consistency_task=True,\n",
        "                                    max_seq_len=104,\n",
        "                                    expected_vocab_size=len(tokenizer)).eval().to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "l3fkJ7KUeWw9"
      },
      "outputs": [],
      "source": [
        "def extract_sequence_fragment(sequence, bos_token, eos_token, tokenizer_eos):\n",
        "  if bos_token in sequence:\n",
        "    sequence = sequence[sequence.index(bos_token):]\n",
        "\n",
        "  if sequence[-1] == tokenizer_eos:\n",
        "    sequence = sequence[:-1]\n",
        "\n",
        "  if eos_token in sequence:\n",
        "    sequence = sequence[:sequence.index(eos_token)+1]\n",
        "  \n",
        "  return sequence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "id": "n7a8lgwRiJzN"
      },
      "outputs": [],
      "source": [
        "def generate_answer(utterance, context=[], max_history=4):\n",
        "  input = \"\"\n",
        "\n",
        "  for turn in context[-4:]:\n",
        "    input += turn\n",
        "  \n",
        "  input += utterance\n",
        "\n",
        "  bos_action_token_id = tokenizer.convert_tokens_to_ids(definitions.BOS_ACTION_TOKEN)\n",
        "  eos_action_token_id = tokenizer.convert_tokens_to_ids(definitions.EOS_ACTION_TOKEN)\n",
        "  bos_resp_token_id = tokenizer.convert_tokens_to_ids(definitions.BOS_RESP_TOKEN)\n",
        "  eos_resp_token_id = tokenizer.convert_tokens_to_ids(definitions.EOS_RESP_TOKEN)\n",
        "  bos_db_token_id = tokenizer.convert_tokens_to_ids(definitions.BOS_DB_TOKEN)\n",
        "  eos_db_token_id = tokenizer.convert_tokens_to_ids(definitions.EOS_DB_TOKEN)\n",
        "  db_null_token_id = tokenizer.convert_tokens_to_ids(definitions.DB_NULL_TOKEN)\n",
        "\n",
        "  input = definitions.BOS_USER_TOKEN + input + definitions.EOS_USER_TOKEN\n",
        "  tokenized_input = tokenizer([input])\n",
        "\n",
        "  input_ids = torch.tensor(tokenized_input.input_ids).to(device)\n",
        "  attention_mask = torch.tensor(tokenized_input.attention_mask).to(device)\n",
        "\n",
        "  encoder_outputs = model(input_ids=input_ids,\n",
        "                          attention_mask=attention_mask,\n",
        "                          return_dict=False,\n",
        "                          encoder_only=True,\n",
        "                          span_task=True)\n",
        "\n",
        "  span_outputs, encoder_hidden_states = encoder_outputs\n",
        "\n",
        "  if isinstance(encoder_hidden_states, tuple):\n",
        "      last_hidden_state = encoder_hidden_states[0]\n",
        "  else:\n",
        "      last_hidden_state = encoder_hidden_states\n",
        "\n",
        "  encoder_outputs = BaseModelOutput(\n",
        "      last_hidden_state=last_hidden_state)\n",
        "\n",
        "  belief_outputs = model.generate(encoder_outputs=encoder_outputs,\n",
        "                                  attention_mask=attention_mask,\n",
        "                                  max_length=100,\n",
        "                                  top_p=0.7,\n",
        "                                  decoder_type=\"belief\")\n",
        "  \n",
        "\n",
        "  generated_belief = belief_outputs.cpu().numpy().tolist()[0]\n",
        "\n",
        "  resp_decoder_input_ids = generated_belief\n",
        "  if resp_decoder_input_ids[-1] == tokenizer.eos_token_id:\n",
        "    resp_decoder_input_ids = resp_decoder_input_ids[:-1]\n",
        "\n",
        "  resp_decoder_input_ids += [bos_db_token_id, db_null_token_id, eos_db_token_id]\n",
        "  resp_decoder_input_ids = torch.tensor([resp_decoder_input_ids]).to(device)\n",
        "\n",
        "  encoder_outputs = BaseModelOutput(last_hidden_state=last_hidden_state)\n",
        "\n",
        "  resp_outputs = model.generate(\n",
        "      encoder_outputs=encoder_outputs,\n",
        "      attention_mask=attention_mask,\n",
        "      decoder_input_ids=resp_decoder_input_ids,\n",
        "      eos_token_id=tokenizer.eos_token_id,\n",
        "      max_length=200,\n",
        "      top_p=0.7,\n",
        "      decoder_type=\"resp\")\n",
        "\n",
        "  generated_resp = resp_outputs.cpu().numpy().tolist()[0]\n",
        "  \n",
        "  resp_tokens = extract_sequence_fragment(generated_resp,\n",
        "                                          bos_resp_token_id,\n",
        "                                          eos_resp_token_id,\n",
        "                                          tokenizer.eos_token_id)\n",
        "\n",
        "\n",
        "  full_context_tokens = tokenized_input.input_ids[0] + generated_belief + generated_resp\n",
        "  context.append(tokenizer.decode(full_context_tokens))\n",
        "\n",
        "  return tokenizer.decode(resp_tokens)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hcrhq3TSYw_e"
      },
      "source": [
        "# DEMO"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "id": "_-WfFOqkW163"
      },
      "outputs": [],
      "source": [
        "CONTEXT = []"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "Ay036u0BVFp7",
        "outputId": "ea5b7653-be9a-4fea-a346-8766633f0d0d"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'<bos_resp> i am sorry, that time is not available. would you like to try a different time? <eos_resp>'"
            ]
          },
          "execution_count": 43,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "resp_string = generate_answer(\"On Thursday at 19:00.\", CONTEXT)\n",
        "resp_string"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d--12aRNX2HS",
        "outputId": "05915644-b5f3-4180-e2ca-ab44eae26023"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['<bos_user> I would like to book a table in the Seni restaurant. <eos_user> </s><pad> <bos_belief> [restaurant] [value_name] Senii restaurant <eos_belief> </s><pad> <bos_belief> [restaurant] [value_name] Senii restaurant <eos_belief> <bos_db> [db_null] <eos_db> <bos_act> [restaurant] [request] time day <eos_act> <bos_resp> what day and time would you like to book? <eos_resp> </s>',\n",
              " '<bos_user> <bos_user> I would like to book a table in the Seni restaurant. <eos_user> </s><pad> <bos_belief> [restaurant] [value_name] Senii restaurant <eos_belief> </s><pad> <bos_belief> [restaurant] [value_name] Senii restaurant <eos_belief> <bos_db> [db_null] <eos_db> <bos_act> [restaurant] [request] time day <eos_act> <bos_resp> what day and time would you like to book? <eos_resp> </s>On Thursday at 19:00. <eos_user> </s><pad> <bos_belief> [restaurant] [value_name] Senii restaurant [value_time] 19:00 [value_day] friday <eos_belief> </s><pad> <bos_belief> [restaurant] [value_name] Senii restaurant [value_time] 19:00 [value_day] friday <eos_belief> <bos_db> [db_null] <eos_db> <bos_act> [restaurant] [request] time <eos_act> <bos_resp> i am sorry, that time is not available. would you like to try a different time? <eos_resp> </s>']"
            ]
          },
          "execution_count": 44,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "CONTEXT"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "rJ5ngOvLYZXd"
      ],
      "provenance": [],
      "toc_visible": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.11.0"
    },
    "vscode": {
      "interpreter": {
        "hash": "5c7b89af1651d0b8571dde13640ecdccf7d5a6204171d6ab33e7c296e100e08a"
      }
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0b446e715bc84ec89e0121ed03c76cb4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0db13575319c43edb7855a45c262c691": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0df7fb8a891e4f90a1b318e770081a5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "16cdf3e4d5414d15b24c016e47ba084d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e4dd40da3cf44fbb906e97adcd55f887",
            "placeholder": "​",
            "style": "IPY_MODEL_ffd8432c713141eda314659ce8f6ef80",
            "value": " 1.21k/1.21k [00:00&lt;00:00, 46.4kB/s]"
          }
        },
        "33e590cba6724cf0b85bfd20dc70692a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fad442dda7814ad0bda08c787cf902fd",
            "placeholder": "​",
            "style": "IPY_MODEL_f47f4e66477f46198e01ae5a9f372e64",
            "value": "Downloading (…)ve/main/spiece.model: 100%"
          }
        },
        "38558746a61d46529c339cc5d36f07a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0db13575319c43edb7855a45c262c691",
            "placeholder": "​",
            "style": "IPY_MODEL_61440360e85944d0910bc2a0a59eef3e",
            "value": " 792k/792k [00:01&lt;00:00, 707kB/s]"
          }
        },
        "419ea03cd0b849f2b321144965745669": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "42b4d88e0a3045f6bb51784cac1102fd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "61440360e85944d0910bc2a0a59eef3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "634086bb039d47be9bfb5b6a8c5f1111": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_33e590cba6724cf0b85bfd20dc70692a",
              "IPY_MODEL_69863c5feb544ccbb8dac7587c85aea4",
              "IPY_MODEL_38558746a61d46529c339cc5d36f07a7"
            ],
            "layout": "IPY_MODEL_42b4d88e0a3045f6bb51784cac1102fd"
          }
        },
        "69863c5feb544ccbb8dac7587c85aea4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fe9a01a140d74334b80e3bea098a9e59",
            "max": 791656,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_852da1a3fe014d01b052d110774342a2",
            "value": 791656
          }
        },
        "6d1219bf2ee64361885ebb8ac399bda7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_84c44fd43f444dc4b65e21c284a45f8e",
            "max": 1206,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_92996c7d79664dfdbf588095fa9e620f",
            "value": 1206
          }
        },
        "721f1a96192e4627b4846e20cc6346f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0b446e715bc84ec89e0121ed03c76cb4",
            "placeholder": "​",
            "style": "IPY_MODEL_0df7fb8a891e4f90a1b318e770081a5c",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "84c44fd43f444dc4b65e21c284a45f8e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "852da1a3fe014d01b052d110774342a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "92996c7d79664dfdbf588095fa9e620f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a733126684fc4f74987b292ab4fb192a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_721f1a96192e4627b4846e20cc6346f7",
              "IPY_MODEL_6d1219bf2ee64361885ebb8ac399bda7",
              "IPY_MODEL_16cdf3e4d5414d15b24c016e47ba084d"
            ],
            "layout": "IPY_MODEL_419ea03cd0b849f2b321144965745669"
          }
        },
        "e4dd40da3cf44fbb906e97adcd55f887": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f47f4e66477f46198e01ae5a9f372e64": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fad442dda7814ad0bda08c787cf902fd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fe9a01a140d74334b80e3bea098a9e59": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ffd8432c713141eda314659ce8f6ef80": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}