Chapter_2_Behind_the_pipeline_.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Chapter 2 Behind the pipeline:.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMMlJRqcgfZW5iMt7hy9SkW",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/JonathanSum/Hugging-Face-Course/blob/main/Chapter_2_Behind_the_pipeline_.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CV25cvN1lMFb",
        "outputId": "37113c89-9d95-40c6-80e1-de0e95a59459"
      },
      "source": [
        "! pip install datasets transformers[sentencepiece]"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting datasets\n",
            "  Downloading datasets-1.10.2-py3-none-any.whl (542 kB)\n",
            "\u001b[K     |████████████████████████████████| 542 kB 10.2 MB/s \n",
            "\u001b[?25hCollecting transformers[sentencepiece]\n",
            "  Downloading transformers-4.9.0-py3-none-any.whl (2.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.6 MB 36.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.4)\n",
            "Collecting tqdm>=4.42\n",
            "  Downloading tqdm-4.61.2-py2.py3-none-any.whl (76 kB)\n",
            "\u001b[K     |████████████████████████████████| 76 kB 4.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.6.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n",
            "Requirement already satisfied: pyarrow!=4.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243 kB)\n",
            "\u001b[K     |████████████████████████████████| 243 kB 45.1 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub<0.1.0\n",
            "  Downloading huggingface_hub-0.0.14-py3-none-any.whl (43 kB)\n",
            "\u001b[K     |████████████████████████████████| 43 kB 1.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Collecting fsspec>=2021.05.0\n",
            "  Downloading fsspec-2021.7.0-py3-none-any.whl (118 kB)\n",
            "\u001b[K     |████████████████████████████████| 118 kB 52.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.19.5)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<0.1.0->datasets) (3.7.4.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<0.1.0->datasets) (3.0.12)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (2.4.7)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2021.5.30)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.5.0)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 32.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (2019.12.20)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.45-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 49.1 MB/s \n",
            "\u001b[?25hCollecting pyyaml>=5.1\n",
            "  Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
            "\u001b[K     |████████████████████████████████| 636 kB 34.0 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub<0.1.0\n",
            "  Downloading huggingface_hub-0.0.12-py3-none-any.whl (37 kB)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (3.17.3)\n",
            "Collecting sentencepiece==0.1.91\n",
            "  Downloading sentencepiece-0.1.91-cp37-cp37m-manylinux1_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 44.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers[sentencepiece]) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers[sentencepiece]) (1.0.1)\n",
            "Installing collected packages: tqdm, tokenizers, sacremoses, pyyaml, huggingface-hub, xxhash, transformers, sentencepiece, fsspec, datasets\n",
            "  Attempting uninstall: tqdm\n",
            "    Found existing installation: tqdm 4.41.1\n",
            "    Uninstalling tqdm-4.41.1:\n",
            "      Successfully uninstalled tqdm-4.41.1\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed datasets-1.10.2 fsspec-2021.7.0 huggingface-hub-0.0.12 pyyaml-5.4.1 sacremoses-0.0.45 sentencepiece-0.1.91 tokenizers-0.10.3 tqdm-4.61.2 transformers-4.9.0 xxhash-2.0.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BOubCj5BeUzx"
      },
      "source": [
        "from transformers import AutoTokenizer\n",
        "\n",
        "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(checkpoint)"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YZjTfNemqCRq",
        "outputId": "12ca916d-4f13-42cd-ec58-e989e443b6f2"
      },
      "source": [
        "raw_inputs = [\n",
        "    \"Happy Sugar Life is all above love\", \n",
        "    \"Love is not about making someone to live for yourself.\",\n",
        "]\n",
        "inputs = tokenizer(raw_inputs, padding=True, truncation=True, return_tensors=\"pt\")\n",
        "print(inputs)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'input_ids': tensor([[ 101, 3407, 5699, 2166, 2003, 2035, 2682, 2293,  102,    0,    0,    0,\n",
            "            0],\n",
            "        [ 101, 2293, 2003, 2025, 2055, 2437, 2619, 2000, 2444, 2005, 4426, 1012,\n",
            "          102]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0],\n",
            "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CAFij0lwq8yD",
        "outputId": "0819b684-0f09-4a9d-baeb-a41ec41d7690"
      },
      "source": [
        "inputs[0]   #It is 13 here because it use the max length of the sentence, which is the second to do padding for inference. I guess."
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Encoding(num_tokens=13, attributes=[ids, type_ids, tokens, offsets, attention_mask, special_tokens_mask, overflowing])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T5PDQvqeqild",
        "outputId": "7a07b95b-4b56-451b-ba72-87fc1b834235"
      },
      "source": [
        "inputs[1]"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Encoding(num_tokens=13, attributes=[ids, type_ids, tokens, offsets, attention_mask, special_tokens_mask, overflowing])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1lFZCd3AeSZm",
        "outputId": "71d5d1e0-5cc2-47e9-bda1-d4f302fed448"
      },
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "classifier = pipeline(\"sentiment-analysis\")\n",
        "classifier([\n",
        "    \"Happy Sugar Life is all above love\", \n",
        "    \"Love is not about making someone to live for yourself.\",\n",
        "])"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'label': 'POSITIVE', 'score': 0.9996786117553711},\n",
              " {'label': 'POSITIVE', 'score': 0.9912222623825073}]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9BYqP7quqWfQ",
        "outputId": "2c5bcdaf-929c-4542-9758-174c6fb8f5ce"
      },
      "source": [
        "from transformers import AutoModel\n",
        "\n",
        "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "model = AutoModel.from_pretrained(checkpoint)\n",
        "import torch"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at distilbert-base-uncased-finetuned-sst-2-english were not used when initializing DistilBertModel: ['pre_classifier.weight', 'classifier.bias', 'pre_classifier.bias', 'classifier.weight']\n",
            "- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cowVMWWJeU1m",
        "outputId": "5fded823-bcc5-42fa-cc37-3573d1b9e52a"
      },
      "source": [
        "outputs = model(**inputs)\n",
        "print(outputs.last_hidden_state.shape)\n"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([2, 13, 768])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BW4ktbG2qYfb",
        "outputId": "42b421ff-c990-49cf-c0a9-bf1c8049e482"
      },
      "source": [
        "outputs[\"last_hidden_state\"].shape"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([2, 13, 768])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lshJXeyep7Sd",
        "outputId": "5733a869-132e-4ee0-88bd-1b82060b1fe4"
      },
      "source": [
        "outputs[0].shape"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([2, 13, 768])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IJrAOySnsjXg"
      },
      "source": [
        "from transformers import AutoModelForSequenceClassification\n",
        "\n",
        "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
        "outputs = model(**inputs)"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uiMbvDgrvgYH",
        "outputId": "3c8f582c-870f-4b68-8266-deea5266c544"
      },
      "source": [
        "print(outputs.logits.shape)"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([2, 2])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wf2ZoeFivwrK",
        "outputId": "8e8f3311-050f-4bb0-a238-5ce41cf873b5"
      },
      "source": [
        "print(outputs.logits)"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[-3.8937,  4.1488],\n",
            "        [-2.3147,  2.4120]], grad_fn=<AddmmBackward>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fwVuDLf2wayF",
        "outputId": "6d726e8b-8fed-4b1d-9d28-30fe674be1df"
      },
      "source": [
        "outputs"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SequenceClassifierOutput([('logits', tensor([[-3.8937,  4.1488],\n",
              "                                   [-2.3147,  2.4120]], grad_fn=<AddmmBackward>))])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KR-pzHIJwcYR",
        "outputId": "4a58c130-f0b3-45d6-d625-3e28ce1bd08c"
      },
      "source": [
        "model.config.id2label\n"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: 'NEGATIVE', 1: 'POSITIVE'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    }
  ]
}