HW2/src/bonus/train_intent.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T7UOl1FH7060"
      },
      "source": [
        "download package"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tnzXC1CZ_JNl"
      },
      "outputs": [],
      "source": [
        "!pip3 install http://download.pytorch.org/whl/cu80/torch-0.3.0.post4-cp36-cp36m-linux_x86_64.whl\n",
        "!pip3 install torchvision\n",
        "!pip3 install pickle\n",
        "!pip3 install datasets\n",
        "!pip3 install transformers"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DvDU0-YQ7-58"
      },
      "source": [
        "upload data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 67,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vUglnJ0yCWcj",
        "outputId": "94475301-a90e-491f-c3d2-3edaea99f5a1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Done\n"
          ]
        }
      ],
      "source": [
        "from zipfile import ZipFile\n",
        "file_name = \"intent.zip\"\n",
        "\n",
        "with ZipFile(file_name, 'r') as zip:\n",
        "  zip.extractall()\n",
        "  print('Done')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i7Kmh6m08NOZ"
      },
      "source": [
        "create dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "q7aG-zO-7zt-"
      },
      "outputs": [],
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "data_files = {}\n",
        "extension = \"\"\n",
        "data_files[\"train\"] = \"intent/train.json\"\n",
        "data_files[\"eval\"] = \"intent/eval.json\"\n",
        "extension = \"intent/train.json\".split(\".\")[-1]\n",
        "datasets = load_dataset(\n",
        "    extension,\n",
        "    data_files=data_files\n",
        ")\n",
        "datasets\n",
        "datasets[\"train\"].features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aTsOfTJt_Ik2",
        "outputId": "28f3717f-33fc-42c8-ee37-e0ff2d063149"
      },
      "outputs": [],
      "source": [
        "for item in datasets[\"train\"][\"text\"][:5]:\n",
        "    print(item)\n",
        "for item in datasets[\"train\"][\"intent\"][:5]:\n",
        "    print(item)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5ngvl43_BlCu"
      },
      "source": [
        "get data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 70,
      "metadata": {
        "id": "3dJGf-mFBrdU"
      },
      "outputs": [],
      "source": [
        "train_texts = [item[\"text\"] for item in datasets[\"train\"]]\n",
        "train_labels = [item[\"intent\"] for item in datasets[\"train\"]]\n",
        "dev_texts = [item[\"text\"] for item in datasets[\"eval\"]]\n",
        "dev_labels = [item[\"intent\"] for item in datasets[\"eval\"]]\n",
        "labels = list(set(train_labels))\n",
        "len(labels)\n",
        "idx2label = labels\n",
        "label2idx = {k:idx for idx,k in enumerate(labels)}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xP33H2c0AIsP"
      },
      "source": [
        "utils class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 71,
      "metadata": {
        "id": "qA4UQQncALJf"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "\n",
        "class ClassificationDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['label'] = label2idx[self.labels[idx]]\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 72,
      "metadata": {
        "id": "H3n-PSgjAQ1g"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "def compute_metrics(pred):\n",
        "    labels = pred.label_ids\n",
        "    preds = pred.predictions.argmax(-1)\n",
        "    acc = accuracy_score(labels, preds)\n",
        "    return {\n",
        "        'accuracy': acc\n",
        "}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qqTYKcllAhA7"
      },
      "source": [
        "train"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1w5pgMVPAgzk"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoTokenizer, Trainer, TrainingArguments, AutoModelForSequenceClassification\n",
        "\n",
        "model_ids = [\"prajjwal1/bert-tiny\"]\n",
        "\n",
        "accuracies = []\n",
        "for model_id in model_ids:\n",
        "    \n",
        "    print(f\"*** {model_id} ***\")\n",
        "\n",
        "    tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
        "    model = AutoModelForSequenceClassification.from_pretrained(model_id, num_labels=len(labels))\n",
        "\n",
        "    train_texts_encoded = tokenizer(train_texts, padding=True, truncation=True, return_tensors=\"pt\")\n",
        "    dev_texts_encoded = tokenizer(dev_texts, padding=True, truncation=True, return_tensors=\"pt\")\n",
        "    \n",
        "    train_dataset = ClassificationDataset(train_texts_encoded, train_labels)\n",
        "    dev_dataset = ClassificationDataset(dev_texts_encoded, dev_labels)\n",
        "    \n",
        "    training_args = TrainingArguments(\n",
        "        output_dir='./results',\n",
        "        num_train_epochs=8,\n",
        "        per_device_train_batch_size=16,\n",
        "        per_device_eval_batch_size=64,\n",
        "        warmup_steps=int(len(train_dataset)/16),\n",
        "        weight_decay=0.01,\n",
        "        logging_dir='./logs',\n",
        "        evaluation_strategy=\"steps\",\n",
        "        eval_steps=50,\n",
        "        save_steps=50,\n",
        "        save_total_limit=10,\n",
        "        load_best_model_at_end=True,\n",
        "        no_cuda=False\n",
        "    )\n",
        "\n",
        "    trainer = Trainer(\n",
        "        model=model,\n",
        "        args=training_args,\n",
        "        compute_metrics=compute_metrics,\n",
        "        train_dataset=train_dataset,\n",
        "        eval_dataset=dev_dataset,\n",
        "    )\n",
        "\n",
        "    trainer.train()\n",
        "    # test_results = trainer.evaluate(test_dataset)\n",
        "    \n",
        "    # accuracies.append(test_results[\"eval_accuracy\"])"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3.9.13 ('adl')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.9.13"
    },
    "vscode": {
      "interpreter": {
        "hash": "ccb1661c9d55548bc588a5cf1cbbb1668e61baa00474ca106e306b6536097a4b"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}