create_char_tokenizer.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ELTAXKPJ1a38"
      },
      "source": [
        "# make vocab file"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "DCsLxkvdkiwB"
      },
      "outputs": [],
      "source": [
        "VOCAB_FILE = 'char-bert-base-uncased-vocab.txt'\n",
        "\n",
        "vocabs = ['[PAD]']\n",
        "for i in range(99):\n",
        "  vocabs.append(f'[unused{i}]')\n",
        "vocabs += ['[UNK]', '[CLS]', '[SEP]', '[MASK]']\n",
        "\n",
        "ascii = [chr(c) for c in range(ord('!'), ord('A'))]\n",
        "ascii += [chr(c) for c in range(ord('['), ord('~') + 1)]\n",
        "pieced_lower_case = ['##' + chr(c) for c in range(ord('a'), ord('z') + 1)]\n",
        "\n",
        "with open(VOCAB_FILE, 'w') as f:\n",
        "  f.write('\\n'.join(vocabs + ascii + pieced_lower_case))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G11UOW8r1v5x"
      },
      "source": [
        "# save tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jdst6_-iJvj7",
        "outputId": "1d2febf2-9688-4509-fd18-094012ce19cf"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "vocab_size: 198\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "('char-bert-base-uncased/tokenizer_config.json',\n",
              " 'char-bert-base-uncased/special_tokens_map.json',\n",
              " 'char-bert-base-uncased/vocab.txt',\n",
              " 'char-bert-base-uncased/added_tokens.json',\n",
              " 'char-bert-base-uncased/tokenizer.json')"
            ]
          },
          "execution_count": 2,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from transformers import BertTokenizerFast\n",
        "\n",
        "MODEL_NAME = 'char-bert-base-uncased'\n",
        "\n",
        "tokenizer = BertTokenizerFast(\n",
        "  vocab_file=VOCAB_FILE,\n",
        "  do_lower_case=False\n",
        ")\n",
        "print('vocab_size:', len(tokenizer))\n",
        "\n",
        "tokenizer.save_pretrained(MODEL_NAME)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AxECKzuv1W7E"
      },
      "source": [
        "# test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WGowDd5HOTV3",
        "outputId": "77e08daf-bee4-466b-ec7e-994d143c7adb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[101, 149, 176, 183, 183, 186, 161, 179, 176, 189, 176, 104, 102]\n",
            "['[CLS]', 'h', '##e', '##l', '##l', '##o', 't', '##h', '##e', '##r', '##e', '!', '[SEP]']\n"
          ]
        }
      ],
      "source": [
        "from transformers import AutoTokenizer\n",
        "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n",
        "input_ids = tokenizer.encode('hello there!')\n",
        "print(input_ids)\n",
        "print([tokenizer.decode(id) for id in input_ids])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "authorship_tag": "ABX9TyP+v4UgPMMNg9sYVMecmBJ7",
      "collapsed_sections": [],
      "include_colab_link": true,
      "name": "character_tokenizer.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.8"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}