BERT/Fine_tuning_BERT_(and_friends)_for_multi_label_text_classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Fine-tuning BERT (and friends) for multi-label text classification.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMiblo1Ci0GTlAbbA5wB3mn",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "9a1aa9f2cc29473f9f8e5459d2641e76": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_308fa6a7348140ec981a8d6c7d31f346",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8bc92587e35443488445e7521fbd0a13",
              "IPY_MODEL_091f8220f33241f288faa0612853585f",
              "IPY_MODEL_1045bb16e3694410898a73cf1b848917"
            ]
          }
        },
        "308fa6a7348140ec981a8d6c7d31f346": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8bc92587e35443488445e7521fbd0a13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_cb95e545fbdd4e99903bf634df694c9f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5080d322a8034924b652b379c04667ed"
          }
        },
        "091f8220f33241f288faa0612853585f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8b1899a0c4b144d7a5e6599f8afb8b65",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 3,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6111a73e684a47769bda7183a836ee91"
          }
        },
        "1045bb16e3694410898a73cf1b848917": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_cd3570ddf67541d7818d97e236c54e54",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3/3 [00:00&lt;00:00, 75.93it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bbba60f793c14100934a268063f63d26"
          }
        },
        "cb95e545fbdd4e99903bf634df694c9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5080d322a8034924b652b379c04667ed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8b1899a0c4b144d7a5e6599f8afb8b65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6111a73e684a47769bda7183a836ee91": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cd3570ddf67541d7818d97e236c54e54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bbba60f793c14100934a268063f63d26": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/BERT/Fine_tuning_BERT_(and_friends)_for_multi_label_text_classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kLB3I4FKZ5Lr"
      },
      "source": [
        "# Fine-tuning BERT (and friends) for multi-label text classification\n",
        "\n",
        "In this notebook, we are going to fine-tune BERT to predict one or more labels for a given piece of text. Note that this notebook illustrates how to fine-tune a bert-base-uncased model, but you can also fine-tune a RoBERTa, DeBERTa, DistilBERT, CANINE, ... checkpoint in the same way. \n",
        "\n",
        "All of those work in the same way: they add a linear layer on top of the base model, which is used to produce a tensor of shape (batch_size, num_labels), indicating the unnormalized scores for a number of labels for every example in the batch.\n",
        "\n",
        "\n",
        "\n",
        "## Set-up environment\n",
        "\n",
        "First, we install the libraries which we'll use: HuggingFace Transformers and Datasets."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4wxY3x-ZZz8h"
      },
      "source": [
        "!pip install -q transformers datasets"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bIH9NP0MZ6-O"
      },
      "source": [
        "## Load dataset\n",
        "\n",
        "Next, let's download a multi-label text classification dataset from the [hub](https://huggingface.co/).\n",
        "\n",
        "At the time of writing, I picked a random one as follows:   \n",
        "\n",
        "* first, go to the \"datasets\" tab on huggingface.co\n",
        "* next, select the \"multi-label-classification\" tag on the left as well as the the \"1k<10k\" tag (fo find a relatively small dataset).\n",
        "\n",
        "Note that you can also easily load your local data (i.e. csv files, txt files, Parquet files, JSON, ...) as explained [here](https://huggingface.co/docs/datasets/loading.html#local-and-remote-files).\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86,
          "referenced_widgets": [
            "9a1aa9f2cc29473f9f8e5459d2641e76",
            "308fa6a7348140ec981a8d6c7d31f346",
            "8bc92587e35443488445e7521fbd0a13",
            "091f8220f33241f288faa0612853585f",
            "1045bb16e3694410898a73cf1b848917",
            "cb95e545fbdd4e99903bf634df694c9f",
            "5080d322a8034924b652b379c04667ed",
            "8b1899a0c4b144d7a5e6599f8afb8b65",
            "6111a73e684a47769bda7183a836ee91",
            "cd3570ddf67541d7818d97e236c54e54",
            "bbba60f793c14100934a268063f63d26"
          ]
        },
        "id": "sd1LiXGjZ420",
        "outputId": "1b5783cd-0e4e-4c92-c67c-57b9288f2381"
      },
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "dataset = load_dataset(\"sem_eval_2018_task_1\", \"subtask5.english\")"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Reusing dataset sem_eval2018_task1 (/root/.cache/huggingface/datasets/sem_eval2018_task1/subtask5.english/1.1.0/c8af6e4accd23f95ac75d14477b0678a4f59d5da34e480ad7de8112ffab04a3d)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9a1aa9f2cc29473f9f8e5459d2641e76",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QCL02vQgxYTO"
      },
      "source": [
        "As we can see, the dataset contains 3 splits: one for training, one for validation and one for testing."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pRd1kXQZjYIY",
        "outputId": "8021590c-5607-4a9c-a474-bc57da503c93"
      },
      "source": [
        "dataset"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['ID', 'Tweet', 'anger', 'anticipation', 'disgust', 'fear', 'joy', 'love', 'optimism', 'pessimism', 'sadness', 'surprise', 'trust'],\n",
              "        num_rows: 6838\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['ID', 'Tweet', 'anger', 'anticipation', 'disgust', 'fear', 'joy', 'love', 'optimism', 'pessimism', 'sadness', 'surprise', 'trust'],\n",
              "        num_rows: 3259\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['ID', 'Tweet', 'anger', 'anticipation', 'disgust', 'fear', 'joy', 'love', 'optimism', 'pessimism', 'sadness', 'surprise', 'trust'],\n",
              "        num_rows: 886\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PgS0wMWExcqP"
      },
      "source": [
        "Let's check the first example of the training split:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "unjuTtKUjZI3",
        "outputId": "6f1e5051-8272-40f9-ced8-ba17a105e904"
      },
      "source": [
        "example = dataset['train'][0]\n",
        "example"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'ID': '2017-En-21441',\n",
              " 'Tweet': \"“Worry is a down payment on a problem you may never have'. \\xa0Joyce Meyer.  #motivation #leadership #worry\",\n",
              " 'anger': False,\n",
              " 'anticipation': True,\n",
              " 'disgust': False,\n",
              " 'fear': False,\n",
              " 'joy': False,\n",
              " 'love': False,\n",
              " 'optimism': True,\n",
              " 'pessimism': False,\n",
              " 'sadness': False,\n",
              " 'surprise': False,\n",
              " 'trust': True}"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6DV0Rtetxgd4"
      },
      "source": [
        "The dataset consists of tweets, labeled with one or more emotions. \n",
        "\n",
        "Let's create a list that contains the labels, as well as 2 dictionaries that map labels to integers and back."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e5vZhQpvkE8s",
        "outputId": "5d513b30-f209-492f-c6ab-245d64a67d40"
      },
      "source": [
        "labels = [label for label in dataset['train'].features.keys() if label not in ['ID', 'Tweet']]\n",
        "id2label = {idx:label for idx, label in enumerate(labels)}\n",
        "label2id = {label:idx for idx, label in enumerate(labels)}\n",
        "labels"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['anger',\n",
              " 'anticipation',\n",
              " 'disgust',\n",
              " 'fear',\n",
              " 'joy',\n",
              " 'love',\n",
              " 'optimism',\n",
              " 'pessimism',\n",
              " 'sadness',\n",
              " 'surprise',\n",
              " 'trust']"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nJ3Teyjmank2"
      },
      "source": [
        "## Preprocess data\n",
        "\n",
        "As models like BERT don't expect text as direct input, but rather `input_ids`, etc., we tokenize the text using the tokenizer. Here I'm using the `AutoTokenizer` API, which will automatically load the appropriate tokenizer based on the checkpoint on the hub.\n",
        "\n",
        "What's a bit tricky is that we also need to provide labels to the model. For multi-label text classification, this is a matrix of shape (batch_size, num_labels). Also important: this should be a tensor of floats rather than integers, otherwise PyTorch' `BCEWithLogitsLoss` (which the model will use) will complain, as explained [here](https://discuss.pytorch.org/t/multi-label-binary-classification-result-type-float-cant-be-cast-to-the-desired-output-type-long/117915/3)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AFWlSsbZaRLc"
      },
      "source": [
        "from transformers import AutoTokenizer\n",
        "import numpy as np\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n",
        "\n",
        "def preprocess_data(examples):\n",
        "  # take a batch of texts\n",
        "  text = examples[\"Tweet\"]\n",
        "  # encode them\n",
        "  encoding = tokenizer(text, padding=\"max_length\", truncation=True, max_length=128)\n",
        "  # add labels\n",
        "  labels_batch = {k: examples[k] for k in examples.keys() if k in labels}\n",
        "  # create numpy array of shape (batch_size, num_labels)\n",
        "  labels_matrix = np.zeros((len(text), len(labels)))\n",
        "  # fill numpy array\n",
        "  for idx, label in enumerate(labels):\n",
        "    labels_matrix[:, idx] = labels_batch[label]\n",
        "\n",
        "  encoding[\"labels\"] = labels_matrix.tolist()\n",
        "  \n",
        "  return encoding"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i4ENBTdulBEI",
        "outputId": "02554a1f-4961-461a-bf29-555b8debeabf"
      },
      "source": [
        "encoded_dataset = dataset.map(preprocess_data, batched=True, remove_columns=dataset['train'].column_names)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/sem_eval2018_task1/subtask5.english/1.1.0/c8af6e4accd23f95ac75d14477b0678a4f59d5da34e480ad7de8112ffab04a3d/cache-e9b1b1258be40e01.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/sem_eval2018_task1/subtask5.english/1.1.0/c8af6e4accd23f95ac75d14477b0678a4f59d5da34e480ad7de8112ffab04a3d/cache-56ad791de205dff7.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/sem_eval2018_task1/subtask5.english/1.1.0/c8af6e4accd23f95ac75d14477b0678a4f59d5da34e480ad7de8112ffab04a3d/cache-0256f120618fb63a.arrow\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0enAb0W9o25W",
        "outputId": "55bc5ba6-d169-49c6-f562-bb7ea4143866"
      },
      "source": [
        "example = encoded_dataset['train'][0]\n",
        "print(example.keys())"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "dict_keys(['attention_mask', 'input_ids', 'labels', 'token_type_ids'])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 137
        },
        "id": "D0McCtJ8HRJY",
        "outputId": "82fb0336-51a3-40ad-ebc0-65eeb7cf4b6c"
      },
      "source": [
        "tokenizer.decode(example['input_ids'])"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\"[CLS] “ worry is a down payment on a problem you may never have '. joyce meyer. # motivation # leadership # worry [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]\""
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VdIvj6WjHeZQ",
        "outputId": "418c14d2-cca3-44e9-d0a2-6ad4ca7a007c"
      },
      "source": [
        "example['labels']"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([0., 1., 0., 0., 0., 0., 1., 0., 0., 0., 1.])"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q4Dx95t2o6N9",
        "outputId": "3ce6c923-0b45-4743-bc4b-7771d088b03e"
      },
      "source": [
        "[id2label[idx] for idx, label in enumerate(example['labels']) if label == 1.0]"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['anticipation', 'optimism', 'trust']"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HgpKXDfvKBxn"
      },
      "source": [
        "Finally, we set the format of our data to PyTorch tensors. This will turn the training, validation and test sets into standard PyTorch [datasets](https://pytorch.org/docs/stable/data.html). "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lk6Cq9duKBkA"
      },
      "source": [
        "encoded_dataset.set_format(\"torch\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w5qSmCgWefWs"
      },
      "source": [
        "## Define model\n",
        "\n",
        "Here we define a model that includes a pre-trained base (i.e. the weights from bert-base-uncased) are loaded, with a random initialized classification head (linear layer) on top. One should fine-tune this head, together with the pre-trained base on a labeled dataset.\n",
        "\n",
        "This is also printed by the warning.\n",
        "\n",
        "We set the `problem_type` to be \"multi_label_classification\", as this will make sure the appropriate loss function is used (namely [`BCEWithLogitsLoss`](https://pytorch.org/docs/stable/generated/torch.nn.BCEWithLogitsLoss.html)). We also make sure the output layer has `len(labels)` output neurons, and we set the id2label and label2id mappings."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6XPL1Z_RegBF",
        "outputId": "22994300-8c93-421e-faa4-678d6cc14aab"
      },
      "source": [
        "from transformers import AutoModelForSequenceClassification\n",
        "\n",
        "model = AutoModelForSequenceClassification.from_pretrained(\"bert-base-uncased\", \n",
        "                                                           problem_type=\"multi_label_classification\", \n",
        "                                                           num_labels=len(labels),\n",
        "                                                           id2label=id2label,\n",
        "                                                           label2id=label2id)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mjJGEXShp7te"
      },
      "source": [
        "## Train the model!\n",
        "\n",
        "We are going to train the model using HuggingFace's Trainer API. This requires us to define 2 things: \n",
        "\n",
        "* `TrainingArguments`, which specify training hyperparameters. All options can be found in the [docs](https://huggingface.co/transformers/main_classes/trainer.html#trainingarguments). Below, we for example specify that we want to evaluate after every epoch of training, we would like to save the model every epoch, we set the learning rate, the batch size to use for training/evaluation, how many epochs to train for, and so on.\n",
        "* a `Trainer` object (docs can be found [here](https://huggingface.co/transformers/main_classes/trainer.html#id1))."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K5a8_vIKqr7P"
      },
      "source": [
        "batch_size = 8\n",
        "metric_name = \"f1\""
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dR2GmpvDqbuZ"
      },
      "source": [
        "from transformers import TrainingArguments, Trainer\n",
        "\n",
        "args = TrainingArguments(\n",
        "    f\"bert-finetuned-sem_eval-english\",\n",
        "    evaluation_strategy = \"epoch\",\n",
        "    save_strategy = \"epoch\",\n",
        "    learning_rate=2e-5,\n",
        "    per_device_train_batch_size=batch_size,\n",
        "    per_device_eval_batch_size=batch_size,\n",
        "    num_train_epochs=5,\n",
        "    weight_decay=0.01,\n",
        "    load_best_model_at_end=True,\n",
        "    metric_for_best_model=metric_name,\n",
        "    #push_to_hub=True,\n",
        ")"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1_v2fPFFJ3-v"
      },
      "source": [
        "We are also going to compute metrics while training. For this, we need to define a `compute_metrics` function, that returns a dictionary with the desired metric values."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "797b2WHJqUgZ"
      },
      "source": [
        "from sklearn.metrics import f1_score, roc_auc_score, accuracy_score\n",
        "from transformers import EvalPrediction\n",
        "import torch\n",
        "    \n",
        "# source: https://jesusleal.io/2021/04/21/Longformer-multilabel-classification/\n",
        "def multi_label_metrics(predictions, labels, threshold=0.5):\n",
        "    # first, apply sigmoid on predictions which are of shape (batch_size, num_labels)\n",
        "    sigmoid = torch.nn.Sigmoid()\n",
        "    probs = sigmoid(torch.Tensor(predictions))\n",
        "    # next, use threshold to turn them into integer predictions\n",
        "    y_pred = np.zeros(probs.shape)\n",
        "    y_pred[np.where(probs >= threshold)] = 1\n",
        "    # finally, compute metrics\n",
        "    y_true = labels\n",
        "    f1_micro_average = f1_score(y_true=y_true, y_pred=y_pred, average='micro')\n",
        "    roc_auc = roc_auc_score(y_true, y_pred, average = 'micro')\n",
        "    accuracy = accuracy_score(y_true, y_pred)\n",
        "    # return as dictionary\n",
        "    metrics = {'f1': f1_micro_average,\n",
        "               'roc_auc': roc_auc,\n",
        "               'accuracy': accuracy}\n",
        "    return metrics\n",
        "\n",
        "def compute_metrics(p: EvalPrediction):\n",
        "    preds = p.predictions[0] if isinstance(p.predictions, \n",
        "            tuple) else p.predictions\n",
        "    result = multi_label_metrics(\n",
        "        predictions=preds, \n",
        "        labels=p.label_ids)\n",
        "    return result"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fxNo4_TsvzDm"
      },
      "source": [
        "Let's verify a batch as well as a forward pass:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "IlOgGiojuWwG",
        "outputId": "cd0b2c99-b520-468d-8ffc-c36211b7820a"
      },
      "source": [
        "encoded_dataset['train'][0]['labels'].type()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'torch.FloatTensor'"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y41Kre_jvD7x",
        "outputId": "b6ca888b-6371-40fb-ab83-3dc24d28320a"
      },
      "source": [
        "encoded_dataset['train']['input_ids'][0]"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([  101,  1523,  4737,  2003,  1037,  2091,  7909,  2006,  1037,  3291,\n",
              "         2017,  2089,  2196,  2031,  1005,  1012, 11830, 11527,  1012,  1001,\n",
              "        14354,  1001,  4105,  1001,  4737,   102,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0])"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sxWcnZ8ku12V",
        "outputId": "26522911-c3cd-466a-ae2d-d81d23003c23"
      },
      "source": [
        "#forward pass\n",
        "outputs = model(input_ids=encoded_dataset['train']['input_ids'][0].unsqueeze(0), labels=encoded_dataset['train'][0]['labels'].unsqueeze(0))\n",
        "outputs"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SequenceClassifierOutput([('loss',\n",
              "                           tensor(0.6442, grad_fn=<BinaryCrossEntropyWithLogitsBackward0>)),\n",
              "                          ('logits',\n",
              "                           tensor([[-0.2136,  0.1645, -0.2064, -0.0940, -0.5121,  0.0718,  0.1167, -0.1767,\n",
              "                                     0.0968,  0.0115, -0.1126]], grad_fn=<AddmmBackward0>))])"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f-X2brZcv0X6"
      },
      "source": [
        "Let's start training!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "chq_3nUz73ib"
      },
      "source": [
        "trainer = Trainer(\n",
        "    model,\n",
        "    args,\n",
        "    train_dataset=encoded_dataset[\"train\"],\n",
        "    eval_dataset=encoded_dataset[\"validation\"],\n",
        "    tokenizer=tokenizer,\n",
        "    compute_metrics=compute_metrics\n",
        ")"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "KXmFds8js6P8",
        "outputId": "66ebb2ab-f93f-48aa-a4dc-36f55f2f8559"
      },
      "source": [
        "trainer.train()"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "***** Running training *****\n",
            "  Num examples = 6838\n",
            "  Num Epochs = 5\n",
            "  Instantaneous batch size per device = 8\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 4275\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='4275' max='4275' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [4275/4275 11:16, Epoch 5/5]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>F1</th>\n",
              "      <th>Roc Auc</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.279300</td>\n",
              "      <td>0.315630</td>\n",
              "      <td>0.690058</td>\n",
              "      <td>0.786618</td>\n",
              "      <td>0.262980</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.237500</td>\n",
              "      <td>0.308757</td>\n",
              "      <td>0.702635</td>\n",
              "      <td>0.795974</td>\n",
              "      <td>0.273138</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.202900</td>\n",
              "      <td>0.319157</td>\n",
              "      <td>0.709615</td>\n",
              "      <td>0.805806</td>\n",
              "      <td>0.279910</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.180800</td>\n",
              "      <td>0.324087</td>\n",
              "      <td>0.702689</td>\n",
              "      <td>0.798627</td>\n",
              "      <td>0.277652</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.157700</td>\n",
              "      <td>0.329103</td>\n",
              "      <td>0.701452</td>\n",
              "      <td>0.798265</td>\n",
              "      <td>0.272009</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 886\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to bert-finetuned-sem_eval-english/checkpoint-855\n",
            "Configuration saved in bert-finetuned-sem_eval-english/checkpoint-855/config.json\n",
            "Model weights saved in bert-finetuned-sem_eval-english/checkpoint-855/pytorch_model.bin\n",
            "tokenizer config file saved in bert-finetuned-sem_eval-english/checkpoint-855/tokenizer_config.json\n",
            "Special tokens file saved in bert-finetuned-sem_eval-english/checkpoint-855/special_tokens_map.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 886\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to bert-finetuned-sem_eval-english/checkpoint-1710\n",
            "Configuration saved in bert-finetuned-sem_eval-english/checkpoint-1710/config.json\n",
            "Model weights saved in bert-finetuned-sem_eval-english/checkpoint-1710/pytorch_model.bin\n",
            "tokenizer config file saved in bert-finetuned-sem_eval-english/checkpoint-1710/tokenizer_config.json\n",
            "Special tokens file saved in bert-finetuned-sem_eval-english/checkpoint-1710/special_tokens_map.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 886\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to bert-finetuned-sem_eval-english/checkpoint-2565\n",
            "Configuration saved in bert-finetuned-sem_eval-english/checkpoint-2565/config.json\n",
            "Model weights saved in bert-finetuned-sem_eval-english/checkpoint-2565/pytorch_model.bin\n",
            "tokenizer config file saved in bert-finetuned-sem_eval-english/checkpoint-2565/tokenizer_config.json\n",
            "Special tokens file saved in bert-finetuned-sem_eval-english/checkpoint-2565/special_tokens_map.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 886\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to bert-finetuned-sem_eval-english/checkpoint-3420\n",
            "Configuration saved in bert-finetuned-sem_eval-english/checkpoint-3420/config.json\n",
            "Model weights saved in bert-finetuned-sem_eval-english/checkpoint-3420/pytorch_model.bin\n",
            "tokenizer config file saved in bert-finetuned-sem_eval-english/checkpoint-3420/tokenizer_config.json\n",
            "Special tokens file saved in bert-finetuned-sem_eval-english/checkpoint-3420/special_tokens_map.json\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 886\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to bert-finetuned-sem_eval-english/checkpoint-4275\n",
            "Configuration saved in bert-finetuned-sem_eval-english/checkpoint-4275/config.json\n",
            "Model weights saved in bert-finetuned-sem_eval-english/checkpoint-4275/pytorch_model.bin\n",
            "tokenizer config file saved in bert-finetuned-sem_eval-english/checkpoint-4275/tokenizer_config.json\n",
            "Special tokens file saved in bert-finetuned-sem_eval-english/checkpoint-4275/special_tokens_map.json\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from bert-finetuned-sem_eval-english/checkpoint-2565 (score: 0.7096149188665537).\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=4275, training_loss=0.20808937385068302, metrics={'train_runtime': 676.8489, 'train_samples_per_second': 50.513, 'train_steps_per_second': 6.316, 'total_flos': 2249123476753920.0, 'train_loss': 0.20808937385068302, 'epoch': 5.0})"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hiloh9eMK91o"
      },
      "source": [
        "## Evaluate\n",
        "\n",
        "After training, we evaluate our model on the validation set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "id": "cMlebJ83LRYG",
        "outputId": "b18102e7-2198-4beb-c874-d39636f740ed"
      },
      "source": [
        "trainer.evaluate()"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 886\n",
            "  Batch size = 8\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='519' max='408' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [408/408 02:01]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'epoch': 5.0,\n",
              " 'eval_accuracy': 0.2799097065462754,\n",
              " 'eval_f1': 0.7096149188665537,\n",
              " 'eval_loss': 0.31915703415870667,\n",
              " 'eval_roc_auc': 0.805805895058838,\n",
              " 'eval_runtime': 4.7187,\n",
              " 'eval_samples_per_second': 187.766,\n",
              " 'eval_steps_per_second': 23.524}"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3nmvJp0pLq-3"
      },
      "source": [
        "## Inference\n",
        "\n",
        "Let's test the model on a new sentence:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3fxjfr8PLD42"
      },
      "source": [
        "text = \"I'm happy I can finally train a model for multi-label classification\"\n",
        "\n",
        "encoding = tokenizer(text, return_tensors=\"pt\")\n",
        "encoding = {k: v.to(trainer.model.device) for k,v in encoding.items()}\n",
        "\n",
        "outputs = trainer.model(**encoding)"
      ],
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8THm5-XgNHPm"
      },
      "source": [
        "The logits that come out of the model are of shape (batch_size, num_labels). As we are only forwarding a single sentence through the model, the `batch_size` equals 1. The logits is a tensor that contains the (unnormalized) scores for every individual label."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KOBosj4UL2tU",
        "outputId": "be370f49-3840-4c76-b193-76083e49701e"
      },
      "source": [
        "logits = outputs.logits\n",
        "logits.shape"
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 11])"
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DC4XdDaHNVcd"
      },
      "source": [
        "To turn them into actual predicted labels, we first apply a sigmoid function independently to every score, such that every score is turned into a number between 0 and 1, that can be interpreted as a \"probability\" for how certain the model is that a given class belongs to the input text.\n",
        "\n",
        "Next, we use a threshold (typically, 0.5) to turn every probability into either a 1 (which means, we predict the label for the given example) or a 0 (which means, we don't predict the label for the given example)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mEkAQleMMT0k",
        "outputId": "fddb51cb-bf01-420a-fd5b-4a7c2e775446"
      },
      "source": [
        "# apply sigmoid + threshold\n",
        "sigmoid = torch.nn.Sigmoid()\n",
        "probs = sigmoid(logits.squeeze().cpu())\n",
        "predictions = np.zeros(probs.shape)\n",
        "predictions[np.where(probs >= 0.5)] = 1\n",
        "# turn predicted id's into actual label names\n",
        "predicted_labels = [id2label[idx] for idx, label in enumerate(predictions) if label == 1.0]\n",
        "print(predicted_labels)"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['joy', 'optimism']\n"
          ]
        }
      ]
    }
  ]
}