Hugging_Face.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "nP4xqPG38Vdl",
        "57w_Foq68DbJ",
        "p_7IPB118GfX",
        "iA5X2H7Q8NMc",
        "vTDroWaz8RqH",
        "gYPRPl3k6-xo",
        "-xyQWy36607y",
        "czPrxhan6qNb",
        "pVDrxc5x8oXW",
        "BNSOj36Mw3dq",
        "xb1wh8FuyHjm",
        "hmQ906x0160F"
      ],
      "authorship_tag": "ABX9TyMXzPPABpN7zL3HCRdB1rUx",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/nnilayy/LLMs-And-Transformers/blob/main/Hugging_Face.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers"
      ],
      "metadata": {
        "id": "R1v8m0oYdzC2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Basics\n"
      ],
      "metadata": {
        "id": "nP4xqPG38Vdl"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Sentiment Analysis"
      ],
      "metadata": {
        "id": "57w_Foq68DbJ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BByFM0aadcQL",
        "outputId": "9465dcd0-e2b7-4d48-81f6-176c877194fc"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'label': 'POSITIVE', 'score': 0.984700083732605}]"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ],
      "source": [
        "# Sentiment Analsysis\n",
        "from transformers import pipeline\n",
        "\n",
        "classifier = pipeline(\"sentiment-analysis\",model=\"distilbert-base-uncased-finetuned-sst-2-english\")\n",
        "classifier(\"horrible piece of nice trash.\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Zero Shot Classification"
      ],
      "metadata": {
        "id": "p_7IPB118GfX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "classifier = pipeline(\"zero-shot-classification\", model=\"facebook/bart-large-mnli\")\n",
        "classifier(\n",
        "    \"This is a shit movie about Goverment the Transformers library\",\n",
        "    candidate_labels=[\"movies\", \"politics\", \"business\"],\n",
        ")"
      ],
      "metadata": {
        "id": "iVf8byZUequc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Text Generation"
      ],
      "metadata": {
        "id": "iA5X2H7Q8NMc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Text Generation\n",
        "from transformers import pipeline\n",
        "\n",
        "generator = pipeline(\"text-generation\", model=\"distilgpt2\")\n",
        "generator(\n",
        "    \"Model distilgpt2 repeats the words after some times, this is because\",\n",
        "    max_length=100,\n",
        "    num_return_sequences=2,\n",
        ")\n",
        "# A perfect example of model Hallucinatuion"
      ],
      "metadata": {
        "id": "OnO1AMcrgANa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Mask Filling"
      ],
      "metadata": {
        "id": "vTDroWaz8RqH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Mask Filling Model\n",
        "from transformers import pipeline\n",
        "\n",
        "unmasker = pipeline(\"fill-mask\", model=\"distilroberta-base\")\n",
        "unmasker(\"This course will teach you all about <mask> models.\", top_k=2)"
      ],
      "metadata": {
        "id": "smFyFAWkg4Nj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Named Entity Recognition"
      ],
      "metadata": {
        "id": "gYPRPl3k6-xo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Named Entity Recognition\n",
        "from transformers import pipeline\n",
        "\n",
        "ner = pipeline(\"ner\",model='dbmdz/bert-large-cased-finetuned-conll03-english', grouped_entities=True)\n",
        "ner(\"hey there, I am Nilay and the Text Generation model above was hallucinating.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5ocSPrRHhrYy",
        "outputId": "0e9ef3f1-8e6f-4798-eacc-7f93a9c7cfca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'entity_group': 'PER',\n",
              "  'score': 0.9974254,\n",
              "  'word': 'Nilay',\n",
              "  'start': 16,\n",
              "  'end': 21},\n",
              " {'entity_group': 'MISC',\n",
              "  'score': 0.56624573,\n",
              "  'word': 'Text',\n",
              "  'start': 30,\n",
              "  'end': 34},\n",
              " {'entity_group': 'ORG',\n",
              "  'score': 0.7065201,\n",
              "  'word': 'Generation',\n",
              "  'start': 35,\n",
              "  'end': 45}]"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## QnA"
      ],
      "metadata": {
        "id": "-xyQWy36607y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Question and Answering\n",
        "from transformers import pipeline\n",
        "\n",
        "question_answerer = pipeline(\"question-answering\",  model=\"distilbert-base-cased-distilled-squad\")\n",
        "question_answerer(\n",
        "    question=\"Where do I sleep?\",\n",
        "    context=\"My name is Sylvain and I work at Hugging Face in Brooklyn\",\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z0MSWTD_jreM",
        "outputId": "edcf821e-11cb-4bed-bff5-084be80e1e4a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'score': 0.48142537474632263,\n",
              " 'start': 33,\n",
              " 'end': 45,\n",
              " 'answer': 'Hugging Face'}"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Summarization"
      ],
      "metadata": {
        "id": "czPrxhan6qNb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Summarization\n",
        "from transformers import pipeline\n",
        "\n",
        "summarizer = pipeline(\"summarization\",model=\"sshleifer/distilbart-cnn-12-6\")\n",
        "summarizer(\n",
        "    \"\"\"\n",
        "    America has changed dramatically during recent years. Not only has the number of\n",
        "    graduates in traditional engineering disciplines such as mechanical, civil,\n",
        "    electrical, chemical, and aeronautical engineering declined, but in most of\n",
        "    the premier American universities engineering curricula now concentrate on\n",
        "    and encourage largely the study of engineering science. As a result, there\n",
        "    are declining offerings in engineering subjects dealing with infrastructure,\n",
        "    the environment, and related issues, and greater concentration on high\n",
        "    technology subjects, largely supporting increasingly complex scientific\n",
        "    developments. While the latter is important, it should not be at the expense\n",
        "    of more traditional engineering.\n",
        "\n",
        "    Rapidly developing economies such as China and India, as well as other\n",
        "    industrial countries in Europe and Asia, continue to encourage and advance\n",
        "    the teaching of engineering. Both China and India, respectively, graduate\n",
        "    six and eight times as many traditional engineers as does the United States.\n",
        "    Other industrial countries at minimum maintain their output, while America\n",
        "    suffers an increasingly serious decline in the number of engineering graduates\n",
        "    and a lack of well-educated engineers.\n",
        "\"\"\"\n",
        ")"
      ],
      "metadata": {
        "id": "UVyYdqf25iTA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Translation"
      ],
      "metadata": {
        "id": "f8KLCfhZ65Ac"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install sentencepiece"
      ],
      "metadata": {
        "id": "XNU-RXBk7NL_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# French to English Translation\n",
        "from transformers import pipeline\n",
        "import sentencepiece\n",
        "translator = pipeline(\"translation\", model=\"Helsinki-NLP/opus-mt-fr-en\")\n",
        "translator(\"Ce cours est produit par Hugging Face.\")"
      ],
      "metadata": {
        "id": "5C5YMe4Y6ktV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Inside the Pipeline Function"
      ],
      "metadata": {
        "id": "pVDrxc5x8oXW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# raw text --> tokens --> special tokens at starting and end --> numbers[vector]\n",
        "from transformers import AutoTokenizer\n",
        "checkpoint=\"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "tokenizer=AutoTokenizer.from_pretrained(checkpoint)\n",
        "raw_inputs=[\"I've been waiting for a hugging face course my whole life\",\n",
        "            \"I hate this so badly right now\"]\n",
        "inputs=tokenizer(raw_inputs,padding=True,truncation=True,return_tensors=\"pt\")\n",
        "inputs"
      ],
      "metadata": {
        "id": "sRiIIBALNjcv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# AutoModel: Thats Model Doesnt have a Head\n",
        "from transformers import AutoModel\n",
        "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "model = AutoModel.from_pretrained(checkpoint)"
      ],
      "metadata": {
        "id": "CuJr-fIUP1zQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "outputs=model(**inputs)\n",
        "outputs.last_hidden_state.shape"
      ],
      "metadata": {
        "id": "X659k_RwpulF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoModelForSequenceClassification\n",
        "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)"
      ],
      "metadata": {
        "id": "bN03_2qlr2XL"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "outputs=model(**inputs)\n",
        "outputs.logits.shape"
      ],
      "metadata": {
        "id": "RE-qzSlWsB5q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# The Model Output doesnt make sense\n",
        "outputs.logits"
      ],
      "metadata": {
        "id": "QtIZWm7GsZCi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# the Outputs are logits and not probabilities\n",
        "import torch\n",
        "predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)\n",
        "print(predictions)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MzurDV_qseYl",
        "outputId": "e7d0b9ff-9146-42dc-9a0b-64c906de322f"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[1.7912e-03, 9.9821e-01],\n",
            "        [9.9951e-01, 4.8962e-04]], grad_fn=<SoftmaxBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# To get the labels of the model we can use the following\n",
        "model.config.id2label"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "myOb967Ns8Tb",
        "outputId": "409da5ec-414f-46eb-c296-d6c133fa33c3"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: 'NEGATIVE', 1: 'POSITIVE'}"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# The output of a teansformers is a vector with three dimensions:\n",
        "# ()Batch Size: Number of sentences/sequences processed at one time\n",
        "# ()Sequence Length: Length of each sequence.\n",
        "# ()Hidden State: Number of features model was trained on"
      ],
      "metadata": {
        "id": "-6cTXdExpOR-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAwcAAAF9CAYAAABRfhTBAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAAEnQAABJ0Ad5mH3gAAC+JSURBVHhe7d17kG3ZfRf21X0e3fcx9955yRqNZuSCSLKE/CiHEtIwCnEqFFWOC4ItDWSsVMI/wXKhVCQTLJEUcbkACbDlKkxZQJFykUgKlmWMjSF/AAVEg6S4KlC2QALz0jwtaWY0c+/cR3efR2d/V589c6an70x33+4+5/T5fEZHfc4+r769917r99177b1Xzl28Z7sAAABLb3XyEwAAWHLCAQAAUAkHAABAJRwAAACVcAAAAFTCAQAAUAkHAABAJRwAAACVcAAAAFTCAQAAUAkHAABAJRwAAACVcAAAAFTCAQAAUAkHAABAJRwAAACVcAAAAFQr5y/esz25DwAALDF7DgAAgEo4AAAAKuEAAACohAMAAKASDgAAgEo4AAAAKuEAAACohAMAAKASDgAAgMoVkmFBvOfBB8r99983eVTKY489Xj7/yBcmjw7nT3/kx+rPP//xn64/F1n+NvkbffozvzCZAgAclD0HsCAebcJAAsFHP/Lh+jOPb0UK6Ucfe2JugsEnf+5nJvcO501NOHjT/W+cPAIADsOeA1gwf//XPle+/wfeW++3BfVnPvOLTbH/7no/BX+2nue5x5v79zUFc57PXoZ2T8HnH/liefjh99WC+mMf/0Td6v7DzePIZ+dxQkie/5Ef/dCL9x9p3hcPNt+V901/ZiRotN/b/h6R3znvzfva3719X16XsJP35fMeeeRLL+4Vaf+tee2nPvPZGmhu9nvm98nfIK97/8MPnYq9IQBw0uw5gAWWYvkDTfGeQjrFcArjtnhOwZxpbdGcQjpBIdPy+gSGT09Cw4MPvqsW23ltCvG8NzIt91Ps536K+7w/gSGfme/PZ+Z7U+RHXp/H08N78t15XwLCToH/R+rr87p8d36H3PJvSTDIZ+c1bRh5uCn2M/3Vfs/WX21CRr4LADg44QAWWArqVorlbDHfSwr4FNdtUZ3Cu5ViPcV/ZKhSXhvttGnTQ5na16VAz2dMy3dNax8nECQoZPhPCv28L3sKpuU7EkJ2QsNOmMi/M+9r3ez3/Gjzb8tz068FAPZPOIBTIAV0uwX/ZoVxXpNCPgV6Cu9WW4zn+QzRyR6F/WrDQor/6aDyWlL0x+4Qkd8h0/K5O0OTdoYbtUON2iBys9/zY5M9Jdl7AAAcXKe/fttPTO4DC+Cxpmhui+orl1948f5K81/uZyjOzrSV8uUv/6ty8eLFFwvt7/rOd9St9h/56P9W35PX5ZZiPc99ugkXeU/8ZvPz8uUrL7vffl/7mXlt+96V5nXT3zut/Z3zGfmZ57/85a/U9yVUZHr7OM/le/K5O+954cUhSvv5PfOe6b8RALB/DkgGAAAqw4oAAIBKOAAAACrhAAAAqIQDAACgEg6AQ7vj9kvle777HZNHAMCiEw4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgWjl/8Z7tyX0ot1+6WO68845y9uyZ0lmVHXl1586dLXfddUd59NEnJlNgb6PxuFy/fr088+xz5fnnL0+mAjBvhAOq9fW18pY3/85ypvn5jW8+U65evVbGTWcOcBRWV1fL+dvOlW973d1NSLhRfuvf/oeyubk5eRaAeSEcUPcSfPd3/a7y27/9jfK1Rx+fTAU4Ht/+7feVe17/uvIbv/GVcv3GjclUAOaBcSOUt33HW8oTTzwlGAAn4mtfe7w88eTXy9ve9ubJFADmhXCw5O69954yGGyVx5twAHBSHn/8yabtGZY3vOH1kykAzAPhYMl92+vuKk8+9fXJI4CT81TT9uQYBADmh3Cw5C5cuM2ZQ4CZeP7y5aYNOj95BMA8EA6WWK/XK7dfulRGI2clAk7ecDgqd95xe+l0OpMpAMyacLDERqNRee655yePAE7ec89ddtpkgDkiHAAAAJVwAAAAVMIBAABQzdUVkldWSlnrdUqvu1q6nZXmcTNhzm1vb5fRaLtsDcf1Nh4vzgWnV1dX68GATz/z7GTKwax2us1ndMvKaqeZeatl/ucWcFRqS7c9bv43atq9YRmPhnX6Qd19153lmWe/VdvSRZE+qt/0VZ3mZ9NVNe3fzvR5lT9tDusYjpp+ajAqw6avOo1SM6x2ei/1S5kxy9QxNTN6u1kn2/Ux6+ZptLq6UvpZB7P+LVCtOGxqxUGz7m026+C8N3dzEQ76vdVy/kyvzuicOGd7e2Wn42nM89+vXRx3lsvt0vz6ZdD8A67dGJaNrdO5Una6/bLaXWtWyF7TADVFwSiFQVMgpFGa67kFHKWmS94pxlZXm4Ks0/zslNFoq4wHm83PweRVp0cKknNNP7W+1qkt3fY4DX/TVy1Is9f2U6urOy31jY1huX5jsDC//6tJIOj2zjT9UrepISb9Ub1NXrBE6jrZzOusl5nTo2Z9HA5uTJ5dbOv9TrMOdkuvs1p28u1L6988z+oXa8XcVrZL8+vXjclXm/VvazCfQX2m4SCJ7+K5fuk2f6lR09Dmtug6TcO7upLGqZTL17ZqSjwNOk0g6PbP1AZ3OBiU4TCdyhK2vMCeUpR0uynSerUQHW5tlNFwc/LsYjvf9FNnm6IkO0fGTT+16E1f5k9CQs4ge/X6oIaERZS9A721c82/J8F01NxO5x6Rw0pASJ2Vn4PN6wu7PmZPXWrFLLfj7Z16cdGlVswte/NSK2YEyjyZWThIArx0vl8Go9MRCnbLTO91tmsyvHrjcLvb50FtfPvnam+ytZktgov7bwFORrbg9tfWSqrowda1hR3ekL0Fly6sZR9JEwpWFz4U7JZiq9MZ1+EOz1/ZmExdDNlg1V8/VwaDoVDwGhIOut1OGY8GTUi4Opm6GM43oTwjS057rfj81a25GnEyk3Bwbr2d2atNCpxMPIXS8Paahndjc1iuXF+8LTNt47u5sdE0wFuTqQD70+v3y9raetnauFpGw8VqQ7LF9faL601BlWBw+oqSadmLUFbG5bnLGwsRgDKEqNtbL4PhsJk3p7iIOGK9XqcG9q2NK5Mp8+3C2Qzj69Za8bQF82kZBpZaMRuTr23MxwbYEz9bUfYYnG9m+NYpDwaRhXlruFoX7qTfRZKGN8OIrl+/JhgAhzLY2io3mjak2z9X25RFkSFSt19YjmAQ+TeuNOVA/s3zbmeIaxM4B4LBQQ0G2TK9Uvrrt+1MmGOpmVI7pYY6zcEgshinJk5tnBp5HpxoOOhkF22GEi3BzJ6W1JsD2XLg9SKoB3Z112unngOOAQ4rY8HTlnSywaFpWxbBxdvWmg57OYJBa9QEoYSiC00fPa8yzLW/fr4OJXLM2+EMhqNmPnfmel1MrZSaKbXTssjinNo4NXJq5Vk70b98DijJuLFlC/uZ6cPm351//7zL2YjSid+4cV3jCxyJnF4xbUr2HqSNmWdn1pvCqdP0U02bvWxGTTG21u/WA0DnUY5/GwwG9hjcogSE3trZJiTM53xOrZSaadlKkCzWqZHnoVY8sSUjpynt1lOVLl+DGzv/7pVydn1+hxeloeitnS+bGwkGDvACjs72eFw2Nm7UNmZei5I4d6bfFJ/z+/sdt+wsztmZ5k1OV5o9Bw4+vnXtWQe7/bOTKfNjp0Y6nQcf70f+3amVUzPP0ol9e85Nu6wzuzXanu9wkFPCDbZyRiJDiYCjl7OdpY3pNm3NPFqbjPfNBcOWVYZSZc9JTjE+TzIMRjA4OjmFZre3VoeSzZPUSKmVlllq5dTMs3RiZyt6/Z1nysbW8m6Naa11x+XZKxv11HHzJLv6u72z5fr14zrNWcaz7vxkt50L9syrnc7DfFtuR7uMnj17vgy3rjXF3nydxS2nLV1d6S51OIhOZ7tsDgbl6rU5mT9NG3Tm3O1lY8PJMY5Szl40Gm7UC6XNg4TSOy+sl82hWnG9Py5ff3Z2F687kXCw1lstF86tLdXBJTfT7YzL9Y1Bc5uv6wX0z1wog62di5sdhfaCSJ3ccqUdXlX+XrnSdA4Azzw4qvlwGPVCVt1uWe00t1xl07EnTGQZzdb/LJ+3crKCLGO9fq9s3ZivUyrefefZMhws31jn3ZrVvmmTRuXZ5+fj2gftxqucoYij0+msNmE4pzZ9YTJltrLX4Ox6rwzVivXUpleubTYhfTZbKk4kHOSaBjkllRnerIyr27VzzQUv5kWKwIwDvn7t1vcaZCxxf2299HrN/B6OmmKivYy9AvO1JCCkGM+Fl+oVLQeb9cJzJ6XXXyv95lZDSjPf8tN8Y1pdPrOcNkVFlo2tehXkwxVsZ8+dL4ONq81yNh8FX84QcselM816Zy9Z9Pvb5ZvPXp88mq2MjV9ZbYrGpk/h6KTP6Td99cb15yZTZqueqaepR5Z9CHpkQ3KukZVrH8zCiVTruZhMWfIxZK3t5u9Q/x5zJOeNzsFJt6rXWyvnzuf8ySt19+9OOFBg7lf+TjneI6fp29oaNMtJrymgbquN5XHKnp18T7bm5nt3rjg6Mt94hazPw2bZyHKS+2trZ8r6eg5qPHibljYnbc+8WG3aZcv8S/KXyIaKebBiD+axyN90ZU7mcaQ2So1EY8a14omEgyx7VuuXZMvbPMlZIG51GMtaUyD0er2yublVC0tuTRrtFOkJWOtnmr9t/3iKqG6v33z+uRdDiQ6Y/crBoQkJpWnPshcgexUOYjgcltUmkM4Lx9bsksJxTv4cuUSbpumYzNHfdd5qo1nKbJllbjuRcGCdnl8ZBpSGN1sBDytbDlMYZDyo4vJoZb6kAOs1RXz2zBylBIO1tfX6+c4CwmElwCZcnjl77kABYTzOxZhyooIT6YYAFsosqymt8pLL8QbppA8rxxdkl2+2OnM82r0I/bW1eqDwUchQpTYYCHTcqoTLdi/XQeSg5py7HoD5IRwsuWy1O+xegxSY2aItGBy/NiBk+NbO8Idbs7Z+pg7rEAw4KgkIOZA9y9Z+pe1ZFQ4A5opwsOQSDg5bIGavQQpMTkYKqQzf6PfXJ1MOJ/Nt57MMJeJoZe9BDmxf3efpi2vbY5wxwFwRDijbhxjZlgIgXboC82QlHPT6/Vvae5C9PeYbxyUBIafE3Y+0PDnmCYD5IRxwKDmYdXTI4UgcXja0Zm9N/v6HkWCQvQaGE3FcEmAz5PAohr8BcPKEAw4lB8ba+jwbGdedPTeHkStWH/YYE9ivLGNZ1gBYPMIBB5atggrM2amF1z7HdO+W95l3HLcaYA+5jAIwW8IBB5ZzmY+3FZizlAL/oBedaod5GFHEcdtu2genKAVYTMIBB5brGriy3WzlmIGDXjzqVs5MBQeRxczVTgEWk3DAoagx58ABay+1GifKAgewkIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqIQDAACgEg4AAIBKOAAAACrhAAAAqBYyHLzxjfeWd7/790we7ci0973vByeP9ifv+cIX/vHk0d4+/OH/8cCfC8BySt+UvmVa24fkufQpe0lftPt9kffe7D3Aze1VFx6mVsx6+9M//fHJo7199rOfekVdusgWMhzcd98b64yYbkgfeuiHyic+8RcmjwDg5L33vX+49lGt6WLk8cefKF/84v9b7wPHK8X6hz70wcmjHXm8exqvtLDDitLAJhC03vWud76i0U2DnNvurTHt1pvpBry11+sB4FY98cSTL+tfcr/dO53npqWfakPFtDZsTH9O+7p81mnaegm3KuvV9HqU9WP3upbnd69T0a6Du6fvtQ6eNgsbDrIFJoEgMye3L33p1yfP7MiehTy/ey9DZmhSY96fLTytPJ/XRZ63GxeAw2j7pdymN0JNF/x57otf/Cf1fl4zXdSnL2r7p+mtnHnNdD/Vvif322EP09Nh2aU2bNe5/MxG5Kx7raxPWf9214pZn9p1r31/5Pn2PdlAPf3cabLQByRnprczKMV+O1PzM7cf+7GPlE984i+Xn/mZn31xJmfo0UMPvb/84i/+7Tq9lc/I52V63nfvvW+YPAMA+5f+Jn1Nbm3fs1umf/jDP177qNzSh8V0/7W7n8p7pqdPb+DK47a/m54O7ATrbDCeXp9S2Ge9a9fBz33ul18ckZKfe9WK7TqY13/2s78kHMyjJMA0gpnhmYGtLATTQ4yyCykh4tUkDORzkhZf68ATALiZFA8pLHJLANhL+qTdwxviZtMjfVv6vHar5u495sArpbhv96ZNr1tZ15588qnJo50RKakFd9eQ0/KeZVgHFz4cZCbunjmZ+W36i+nXTO9hmJbnc0uj3t4A4Dikv2kLlmk366MifV62brZ91PRGMWBvWaf2KuSzPmWjcCv385rd6+D0/Tw3XStmD8JptNDhIJIIs2tnWmb4zvRP1WMHkvLaFJiGNQtJprdjNyPPt2kwu4naXUVZEF5rr8Miefvb3jq5Nz/uv//el90OIq//5M8efk/PD/83P1je83v3Hp/7937l/6w/8x15HYvtoMsWp98s28P0WylGdvdF2biV/iinNs1z6a9a2QuRPirTczutQxruvvvOepsXe7UdR9WetP3MMjqpeZx1KutOWwe2Ut/l1q5TCesJ3Hl9asW91sHUllnvdq+DNwv7i2rl/MV7tif3j82l2/qls9oto/HKZMqtaVPc9O6hyIyZnvl5nNdm2vRrM61dCG72nlcbpnQrVps/QWd1VJ5+fmMy5WT9mf/lT5av/OvfKp/7pV+tj3v9s2XULAGDra36eD/6a+tlZaVTRqPRZMqt+Zf//B+XR5sVtPWZv/XL5dP/1/62iKWw/8if+hPlv/pD/+1kysF89E99sDz2+JOv+L40/H//73yqvON7v69+Rx7v93c6Cb1et2xtbZTRcDiZ8to6nU5ZWz/bvG8wmbJc0gl/4IMfKY89tveQDY7OyspK6fd75drVK5Mpe+v110pnZbtpf65PppycFCYf+B/+2IvtYb/fKRfOrZXh8Nb6qfQfu/ubyLTp+9H2RW3/svt92TCV4iV2vyeP2/dNf8bu7zisXm9cvnW5aWPSQZyw3/efPVDe+4N/sPzkn/tL5emnny399dtq/TAejyevOFnpJ+Jjf/GlsedXnvmtcuGut0weHV76v/Qzs7K+1i83rn1r8uhktfP5g//TzkiNuy+tN/O5U8ZHtMjdbF3I9On1qd0AvLvOa5/bzzrYvnb3ZxxWZ3W7+VsMy/Mv7L82O0oLGQ4W2azDwe4OcV7Cwff/1+9/RdGWgjzT2p/ttDc1K+Dn/9nOCtiGgxR909Nbu1/faqc/+Hvf+bJw0E5PWGnDwbT2d3mtz837p/89+T13v/ZWCAcH92rhoJ2vrVd7nPsx/Tj387NdJqbvL6NFCAcx3R7+6t/9e0cSDk6LWYaDeO8P/cHy+97zQA0Il5sCad7Dwe52Ifaa1mr7hGUOB9HO5wSEow4Hi0w4WDJtOHj7d33vZMrJu/vuu+rK+E8//4XyK3/3H85tOGj3KGR6GtJP/62/XX74j/5gndbuXcj0dlhRGto8bj+rfa6d3jbAKRLbwi3TP/aXfvZln9VOj3bPwcN/9A/XwjKdRAJFGwCi3WuRz33kn/16/R0jr4/2M6OddquEg4PbKxyk804IzLKV+frxv/hXXhEMM6TswQfeWd+beZnlIa+PFAvt8+2ylM8/jnm+SA4SDr7t7tvLW/6T+ydTTl7bHv6dX/218hv/4p8LBxMJB+9o+qnxDCu1bFWOj/3UJ8s3vvnc3IaDtAEf/Z8/+LJ2oW1b0g7kftqWtm9p24dMT18y63Dwzv/0OyaPZqOdzz/5Ez8hHEwIB0sm4eA73vo7ynd/7++eTJmN7M77p//PF8rf+PlfmLthRW2xnekprNKItkV5nmsb3bZwT0PbNq55XQr0vGe68W6n570p9NvvyPR2z8H09+3+julwcP9999b7N3tNpud3yndMv/4oCQcH91rDijIP2yFqmX8JoFkWcv+RL+wcyNaGhMj03E9hML1Mta+fp2FoJ+0g4eA/f887y1vf/O2TKbOR9vCXf+XXyj/6B/9AOJhIOPi+/+L3lzvvnN3Y/7vvurO8/e1vLR/+8T8783CQ/mda2ou2f0nbMr3+727v00akH0houFk/MysJB//9+//Q5NFsZD4npP+Vn/3L5emnLwsHDeFgycx6WFF84I//sboy/uSf+6m5HlY0PX26cW2fa4vy6WMO2q24KezSSKcBjjTCCQcJAtOfMx0OpsPEq4WD2P17TL++/R3y+kzPFqV8RruH4igIBwd3s3DQBrgsB9MBNMtP7rfzuA0B7fvzmmwNzM/p0NC+N/b6vmWwKMOKIsdhPf3Ms+V///n/w7CiKbMeVpSDxf/M//ony0/+2Z8q/+4/PlXrh3ncc3CzdiF9T97X7u3O82kPdg9HatuXWZn1sKLM59Qkn/xrP1+e/u1Hm/lsz0HMOhws/NmKOJjpYHAapUhLkdfuiUhjnFuKvDTsaaB3bwFqtVtyDiOf225lnt5T0HYICTnpQBISmB+ZH+38aocEROZbbun424CZxzG9TLXPTcvr8lxek8DI/GqDQQoT5sd0MPjKV//NZOp8ulm70G54SuGfjVWtPKcf2DEdDOZ9Pi+bhQ8HOUKc/csKOI/BIFtXUojldpiGM++pW2ma99c9BE2DnWIvxXr7XCQ0ZIxnHre3Vhrwdit/u+V3vxIqcstn5Ptb7ffH9HEKzMbu5awNg/m5e55naFCWh7Zjb+ddlpm8N5+xl+l5ftiwyfHLwcg5EFkwmE+LEAxid7vQ9inZ6FBvTRuQjVattCfZ253XZm/mshMM5lOnv37bT0zuH5v1tU5ZXVkt29tHu7s255d94IF3Hdmpo1oJHH/gD/z+8pWvfHUy5eisNH+C1ZXtcn1j/8NBjtKjjz4+ubej0+mV7MEbH2CIUKfbbf4dmZ9Hs+/v8pUXysWLF168leZv9OV/+dU6PVteLl9+obl/pVxpfuZx+568Jq+9MvX+bLmp0xt5bYY3fOc73lYfp2jPZ/3a//0Py3dNpuX1K81/eW3ed+nShfr6emDpZFq+I5+T++1rd/8e+e58Zt6b24/+yH9XHn/8qZ3v+s631Y7gk3/tb774u92qTme1jEbDsn2A3eyrq6ul2+0175vNrvlZyzxsl5PcMg9/M/O0mZ75k3meZamdR1nm0tFnGamPm2UngeFS897M43be53Xt8hFf/ldfLT/w/f9lXR7y3rxv2eTvkWFsg63NyZS9dTrdOtRyPDr5oW7Xr994WVGSdWqt3y3jYx7+mlMg5pSHr3Wq0fc1/dtx9EH71elslxubTRszgyEe2ZuTW6vTXav1w1H1OQe1u91vtX3KdLvQTk87kDYgt5/763/zxTYit7w+bUTanLoxa9fnnqRut1OGgxuTRydr93w+t95t5nFTW0wen6T9rpev5ijX2bSL29vjsrF1NMO3D8oxB3vIQpILoh3HVZLn4ZiDafNwzMFp0G4tasektluVj+ugVMccHL/Mw2z5mx5nzP6kEFqUYw5aR3Wdg9eSAiJFyGtdWTUXRnvoofdPHp28WR9zMG3W1zk4zWZ9zMG0WZ7KdL/r5avZufDuj99SwGg55uAWZGbmCnWRn7nKXa5olxmU5yKFfqZnWp7L/XYoUqa199vPyuNcDe+hh36oPg/7kSFMKSQzpCS3HF+wzGerWXTZ3S8YcBSm+5/0L5/4xF+oV0Zu+5f0O22/1fZn7Wszre3L2s+Z7sOm39u+Dtip6bJu5NauV5HHrem6b/d6mfUst/Yz2vWrrSlbeX3en+fzXD5n+vlFtfDHHNx77xsm93Y88MD3vXh561ZmWLbA5Ln2cbQNbCuflcSXqyPn0vaz3GrDYskxDhlC0t5yUBqLqz2AHW5FCo8vfenXX+x/0r+kf2qnRfqaPJctjtljHXkur83P9EcpNtr3fO5zv9z8/KH6ury+fe/0Vf1hmaXGSw2YdSPrTIr+tu7L3oFWe3+v9TLP5XH7GdloHLvrxvZx1r9cSTnr4nGMOjlpp+psRZmRkWMQ2gUhpnfx5DVZUADgOKXwn946eTN5PsVHCpK9Xpf+LM/ldQkEbR+WoJCtmtP9HSy7rA8p1CP1X+q+w6wjbe2Yn7kt03q2dKcyTQP75JNPTR4BwPFIQZEtiSlU2uEKu2V6Nmhla2Nb0OyWz9n5jF+qWzjbrZsZH91uGZ3eWw68ZPcIk8NoA0cbGE67pQgH2RLT3rLVpT27UX62u2enG9bM/HYLzl5bcQDgtbRbGjPkIP1K+pP0O22xkuczvd3bnX6nNd33tHu828Kk7Zva99sjDi9JiM66kVu7nkzXfan3Mn16ncn03etQ3hcZ1pfns64lILTT83N6nc3z048X2cKHg3Yo0e5E1y4IreyyzS1bZ9rnsgUmC0NmfBrv9rPyfD4vB5YAwGGlH8nQn/QvbXGRvdeZlvuZnr0HKTRS1ERelyFD09PznjzO57UFyM5zn6qPT8M4ZzgKWX+yRy0bg7O+7K77Eg6yYTj3sw5Guy5mvWyDeUJ81q+se+3eur3WzVY+L3Vmhv8tulN/KtPMvMysdsbOmlOZchScypR55lSmi8+pTJeDU5nuLcV/iv3dG5pPilOZAgAAc+HUh4OkvhwQBgAAryV146z2GsyDpdhzMH0sAgAA3Myy142GFQEAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHAAAAJVwAAAAVMIBAABQCQcAAEAlHHAoKyuTO8zO9uTnPh3w5QDAEhIOOLDt7ZSZ0sEsrTTpbGc+HEDzenONk3Lg5ROAuSAccGDb47E9BzO2Ew7Gk0f7M858W7XKc/xWV1fq8gbA4lEpcGCj0ajp/C06s9LuNTjMllnzjpOQZXQ0Gk4eAbBIVAkcWLZY56bInI1O83cfDQeTRweTgi1bdeE4pW0YDYUDgEWkuuNQBoOt0ulYfGYhf/fB4HDhYFjnW2fyCI5egkH2ao3Ho8kUABaJ6o7GwbckD7a2ahGQ4QOcnASD0Xh86MIr48CzRbcrIHBMugmvW5uTR69O6wEwf4SDJZctfIct8Lc2N0u3q8g8KZlP3W63bG1tTKYcTt7f7XUFO45cwut2899wv8PemmXwoAfWA3C8hIMlt709KquHLBIHg80aLgxTORkJYhnONR7d2nCN7D3Y3NwoPcGOI5Q9ib1er2xu3JhMeW1pe4QDgPkiHCy57XETDm6huN/cuF63Fjr+4Hj1et16Ctmtpqg/Chn2MWrmfT4XblX2QmVZ2rhx/UCnME3b49gEgPmiolty6ZhXVg5/7ED2HNy4ca3uPTDE6HjUYLA9LhtNEDtK2cKbQi6fb4gRh5U9Bv1+r+6N2vdwokaWudyygQKA+XEy4cCFMl/mMOenP06j0aB0ur3Jo4PLFu0b16/mXuk3hWaKBW5dAtfaWr+efjRbZI9D9vxk/qe4MzyMg8oGgQxPy/KZM2EdRLdpc7LszY3aLuusXpIhX5O7c8D2i2MyR3/YeauNZm6Gf44TqeJG421npZhYWdlu/h6TB3NiPNxsOvhbG16SlToFwtYgn9UUDE1IMNTo4LIlNUV6ivVcj6D+TY9oKNHN5MDy7P1JptsJCc5Cxc1luUwoSHDNHq1r167WAHtQCQfj4cECxXFKu2yxf0n+FuOm754HO1flN3OOWt1zN0fH/OysgwJCZGlP7TwrK+cv3nPs377e75TbzvbLYKRY7HbGZWNzWK7emKMtZo31s7c3BWLGCx/NLv5ur9eEhH7pNKEjQ1dsEXhtaahzy6lGc+DxLK4wW4eH9daawi1DmSZXYc6s0y8vtSwGCQU7xcTO2YhyOuPDFhY51uDM+pmycf35yZT58Lo7z5atLQt76vBO01c989z+Dy4/Tp3uWun0zjTtogvrHaXs5c8O460bVyZTZuv8mV5ZX+uWoVqx9Jr174XrW2VjazbDLk8kHKRTufvSetkcmOH97rg8f3WzbA3ma/dBt7deVlZ7ZeMAZxrZrxQCq7dwXMMySJGVEHWQgzmPW72ORXPLvGO55fSk2+OdC5sdRdBfP3O2bI+2ynBwvHvFDuqOi+vN/+cg6Z3Hy2p1dbuMRqNyuemr5kHaobUzF8vm5nxtVFt02QO4PW7Ww635CIH93mq5dH6tbA31OWu9cXn6+Y2mLTr2En1PJxIO4q6La0330imj8fIWiKmN+00a/MacbI3Zbe3spTqEZTi0dQY4HhlO1F9bK5tzttcgzq53y5n1HOez3Bsyut1xuXJ1q2wN5udg8bUzl8pwNF8bUBbd2lqvbN64MlcnBfi228+UrVGusj6ZsIQ6TThfKaPyzOXZhfMTi2fXN4ZldcnHknVWx+X65vwW3sPNa02nvW4LP3As0rakjRk0bc08utG0zxlmscxNYI49inkKBjEabtTjoTga+VvWIb9zdraw1EiplZZZauXUzLN0cuFgMwvgdk1Ey2i16Wy6zV/76o35DQc5c0gOEFxbPzOZAnB00rbkBAjjeTpL0ZRsrawbspa0n4rVpjC7NmfHxEWGoCVcOhve0eg2KXi4dTxnwbsVqZFSK6VmWkY7NXITDmrNPDsnupZduTZoZvpyNrqd1VF54frgSMbrHqdBbSy2m048Y28BjsZ63egwnrQx8+vqta2mCN1eyoCQf3PGON+Y8VbLmxluXXNl9yOQYw1G4+FchvTUSKmVUjMto9TIqZVn7UTDweZgVG5sjepR2MskZygajrbLtTltcHcbbLzQpPaVSWcOcGvSlmSozmAj10OZfzkQN2d3XqbhRdkgn3/z5Rfm4yDkvYyGOYvbliu734IMJ8pJYuZ5XUytlJoptdMySW2cGjm18qyd+P65K9e26kFFyxIQuqs5jee4PDfHDe5etpqAkK18Z86eaxoSW2qAg0vbkTYkbclOm7IYBoNxeaHpq3q97EGYTDzFdoLBdnn+ysZMz62+H/V4le1RM2/0SweVYJC9Bjvr4nzP59RMqZ1SQy2D1MSpjVMjz4MTO1vRbhfP98tas3LnfLZz3hYdSrY4ZaEejcflW1cWKxhMyylOe2tn64WytrYW998BnKx+f63etrauldFgMduO9bVOuXB+reQEbuNTeqa91c7OsYDPN8VYQtGi6K2db373bvM7H83pdU+7hILsMUgwmLeDkF/NHRfWmuVztQzHp/MMRvV41CYYZG/B5avzc1HImYWDOLfeLbed7TUBoZTR9umZ8Wloe02Dm11jGTu36FZWVku3f7Z0Or16ca7cNMbAbjlgtNfrl16/X0bDQT3gcZ6uwHoY3e5qExD6dajluClQTsuZNLO3IAcfZ09BhhLNy9WQD6JuvGr6puFoVEYjF9vcS72wZWe1HmOQIcOLKHVi6sXBaKX5d5yOkJ4NyJ2VcTNvSq0T523Y+UzDQaTBzYw/s5YDZHIwStMA19vkBQsgMzmnnsotp8Hb2ByVF24MmsbqdDVUGSLQaRrjXK1yXBvjYXMbNZ3KztApYLnUi+Q1t07ahm63thGj4WYZDXLxntN1QOGZpjg5d6ZX76ef2rnlfp0099JP7dx2hkplr/a1pijZnNEVWI/KzsarM/XK7gkICTk7fdLp6n/3KwF956xOKzUYjJtQkIuczesZwvar02lqxXoF5aaNaRbZnTpxZx1cFNlLkDox62DOynsjtWKzDo7n8B8x83Awbb3fqbdeN51NWrHJE/Os+eulMRo0jVIuc7251TROS9AorXZ6dU/CSqcpCFaaRFQbpMmTwKlXm7nm/+pVk5sCpJ4KecELkP3o9Vabfqpbf6afSiG2CFIs5yDPXL8ggWA4PH0bdLLharXbawrjbjNflvGq/Nt1PmfjXdbFHLy9SEOI9iMblNf6WQebWnFycPWi1IrZSzdo1rvUirnNs7kKBwAAwOwswXkYAACA/RAOAACASjgAAAAq4QAAAKiEAwAAoBIOAACASjhYYrl40d133Tl5BAAsk9QA6gB2Ew6W3O96+1sn9wCAZXLu3Nly223nJ49gh3AAAABUwgEAAFAJB0vu8SefmtwDAGDZCQdLbDwel/ve+IZ6YDIAsFy6vW4ZjoaTR7BDVbjkXnjharl06cLkEQCwLC5euK1cfeHa5BHsEA6W3De++Ux5wz2vnzwCAJbB7Zculn6/X7713POTKbBDOFhyTzzxVDlzZr3ce+89kykAwGnW6XTKm9/8O8rXvvbYZAq8RDigfPVf/9vy7W+6r9x/372TKQDAaZRrG3zPd7+jPPvsc+W3v/7NyVR4ycr5i/dsT+6zxM6ePVPe8ubfWbrdTvnmN58pL1y9Vg9YBgAWWy50trW1Ve64/VJ5/etfV/79v/9aefwJZytkb8IBL3PnHbeXO++6o5xrwoKzGAHA4svegueee748f/mF8o2vf7NsDQaTZ+CVhAOAJbWyslK+93u+s/x//+I3J1MAWHY2DQMssQsXbpvcAwDhAAAAmBAOAACASjgAAAAq4QAAAKiEAwAAoBIOAACASjgAAAAq4QBgiW1sbEzuAYArJAMAABP2HAAAAJVwAAAAVMIBAABQCQcAAEAlHACcUvfff9/Lft6qV/uco/oOAGZLOABYQJ/8uZ958bZXYf6eBx+otzz30Y98eDL11uRz9vquo/wOAGZLOABYMG2B/oEf/VC9PfbY4zUIRJ7bq4Cf1gaHNjz88MN/ZPLMzvvzuP282P2ayLTp1wBwOggHAAvoTZPivC3wH374fXV6Hue5m8lr270Nec/7H36oef0by5/+yI/V5/9q81y858F3v/j5mZYA0n5uprV7Edr3AXA6CAcACyrF+WF8/pEvlE9/5hfq/U995rP1dl8TEFL0P9qEgDyXaTsB4d3lR370Q/U9eS4SKh5/7IkaKh5sngfg9BAOABZQW8SnaI9X21vwWrJXIPKZ03sHHm0CwOcf+eIrPjvBINP//Md/unz/D7x3MhWA00A4AFhwKe4//ZlfrEN8sjU/Uug/+OC76v2bBYcU+dPyOY80RX+GHWXYUMJHPid7DzKtlb0K2XuQ70uIiPYnAItt5fzFe7Yn9wEAgCVmzwEAAFAJBwAAQCUcAAAAlXAAAABUwgEAAFAJBwAAQCUcAAAAlXAAAABUwgEAAFCtnHOFZAAAoGHPAQAAUAkHAABAo5T/HyJMII9fAIvUAAAAAElFTkSuQmCC)"
      ],
      "metadata": {
        "id": "8hqq7shYrFn3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Model Head\n"
      ],
      "metadata": {
        "id": "RSo_gm88q9-Q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Instantiating a Bert Model\n",
        "from transformers import BertConfig, BertModel\n",
        "\n",
        "# Building the config\n",
        "config = BertConfig()\n",
        "print(config)\n",
        "# Building the model from the config\n",
        "model = BertModel(config)"
      ],
      "metadata": {
        "id": "hYPx-Q8rvKyx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Loading Methods"
      ],
      "metadata": {
        "id": "BNSOj36Mw3dq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Creating a model from the default configuration initializes it with random values\n",
        "from transformers import BertConfig, BertModel\n",
        "config = BertConfig()\n",
        "model = BertModel(config)\n",
        "\n"
      ],
      "metadata": {
        "id": "ZIfCQXEDw6GN"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "The model can be used in this state, but it will output gibberish; it needs to be trained first. We could train the model from scratch on the task at hand, but as you saw in Chapter 1, this would require a long time and a lot of data, and it would have a non-negligible environmental impact. To avoid unnecessary and duplicated effort, it’s imperative to be able to share and reuse models that have already been trained."
      ],
      "metadata": {
        "id": "XBI0LQfhxOex"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import BertModel\n",
        "model = BertModel.from_pretrained(\"bert-base-cased\")"
      ],
      "metadata": {
        "id": "ZwLmlu6vxWBa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Saving a Model"
      ],
      "metadata": {
        "id": "xb1wh8FuyHjm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model.save_pretrained(\"directory_on_my_computer\")\n",
        "\n",
        "# This saves two files to your disk:\n",
        "# (1)ls directory_on_my_computer:  configuration is necessary to know your model’s architecture\n",
        "# (2)config.json pytorch_model.bin: model weights are your model’s parameters."
      ],
      "metadata": {
        "id": "3UjW-NupyOmL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Inferencing  "
      ],
      "metadata": {
        "id": "fc4pLN31y6VT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# sequences\n",
        "# AutoTokenizer that\n",
        "# Then feed to model"
      ],
      "metadata": {
        "id": "JWyaUK4py-Yv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Inside Tokenizer API"
      ],
      "metadata": {
        "id": "hmQ906x0160F"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# UNDERSTANDING OF TOKENIZERS AND TYPES OF TOKENIZERS\n",
        "# Tokenizers have one function, they convert raw text into numbers which can be fed into a model\n",
        "# There are three methods of tokenization\n",
        "\n",
        "# (1)Word Based: Splitting based on words, punctuations etc\n",
        "# Issues: Too large of a vocab size\n",
        "# Resolve: Reduce The Vocab size\n",
        "# Issues With The Resolve: Many Words are recognized as out of vocabulary, leading to loss of information\n",
        "\n",
        "\n",
        "# (2)Character Based: Splitting Based on Characters\n",
        "# Advantages: Reduced Vocaulary Size\n",
        "# Issues: But the meaning of each characters contains very less meaning compared to a word\n",
        "# Issues: Since Splitting is done on each characters the sequence length gets very big, even for small sentences,\n",
        "# Hence the model input gets very limited\n",
        "\n",
        "\n",
        "# (3)Subword Based:\n",
        "# Rule-1: Frequently used words are not to be split\n",
        "# Rule-2: Rare Words are to be broken down meaningfully\n",
        "# This resolves to some exctent the issues of bigger vocabulary\n",
        "# This also preserves the meaning's of the word.\n",
        "\n",
        "# There are also many other Tokenization methodologies used, some examples are:\n",
        "# (1)Byte-level BPE in GPT-2\n",
        "# (2)WordPiece in BERT\n",
        "# (3) SentencePiece or Unigram in several multilingual models"
      ],
      "metadata": {
        "id": "Z8BQ5f0t1_4n"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Loading a Tokenizer\n",
        "from transformers import BertTokenizer\n",
        "tokenizer = BertTokenizer.from_pretrained(\"bert-base-cased\")\n",
        "\n",
        "# or General Method\n",
        "from transformers import AutoTokenizer\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-cased\")\n",
        "\n",
        "# Using the tokenizer\n",
        "tokenizer(\"Using a Transformer network is simple\")\n",
        "\n",
        "# Saving a tokenizer\n",
        "tokenizer.save_pretrained(\"directory_on_my_computer\")"
      ],
      "metadata": {
        "id": "jcgcOm1t_FJg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Workflow of Tokenizers\n",
        "# raw_text-->tokens-->add special tokens-->input ids\n",
        "# Understanding the Tokenizer API better\n",
        "from transformers import AutoTokenizer\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-cased\")\n",
        "sequence = \"Using a Transformer network is simple\"\n",
        "\n",
        "# Tokenizing given words\n",
        "tokens = tokenizer.tokenize(sequence)\n",
        "print(\"Tokens are: \", tokens)\n",
        "\n",
        "# Converting tokens to numeric ids\n",
        "ids = tokenizer.convert_tokens_to_ids(tokens)\n",
        "print(\"Numeric ids are: \", ids)\n",
        "\n",
        "# Converting numeric ids to tokens\n",
        "decoded_string = tokenizer.decode(ids)\n",
        "print(\"Decoded numeric ids are: \", decoded_string)"
      ],
      "metadata": {
        "id": "TtlfzWpWBO-O"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Tokenization API"
      ],
      "metadata": {
        "id": "48TxCj2cGGCn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#  In the end you dont have to worry about anything, The Tokenizer API Will take care of everything\n",
        "from transformers import AutoTokenizer\n",
        "\n",
        "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
        "\n",
        "sequence = \"I've been waiting for a HuggingFace course my whole life.\"\n",
        "model_inputs = tokenizer(sequence)\n",
        "model_inputs"
      ],
      "metadata": {
        "id": "tX-FI3HoDAye"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Different types of padding available\n",
        "# Will pad the sequences up to the maximum sequence length\n",
        "model_inputs = tokenizer(sequence, padding=\"longest\")\n",
        "\n",
        "# Will pad the sequences up to the model max length\n",
        "# (512 for BERT or DistilBERT)\n",
        "model_inputs = tokenizer(sequence, padding=\"max_length\")\n",
        "\n",
        "# Will pad the sequences up to the specified max length\n",
        "model_inputs = tokenizer(sequence, padding=\"max_length\", max_length=8)"
      ],
      "metadata": {
        "id": "gZljLz5MGXyY"
      },
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Return types of inputs\n",
        "sequences = [\"I've been waiting for a HuggingFace course my whole life.\", \"So have I!\"]\n",
        "\n",
        "# Returns PyTorch tensors\n",
        "model_inputs = tokenizer(sequences, padding=True, return_tensors=\"pt\")\n",
        "\n",
        "# Returns TensorFlow tensors\n",
        "model_inputs = tokenizer(sequences, padding=True, return_tensors=\"tf\")\n",
        "\n",
        "# Returns NumPy arrays\n",
        "model_inputs = tokenizer(sequences, padding=True, return_tensors=\"np\")"
      ],
      "metadata": {
        "id": "Me-yscXqGqHu"
      },
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sequence = \"I've been waiting for a HuggingFace course my whole life.\"\n",
        "model_inputs = tokenizer(sequence)\n",
        "print(model_inputs[\"input_ids\"])\n",
        "print(tokenizer.decode(model_inputs[\"input_ids\"]))\n",
        "# The tokenizer added the special word [CLS] at the beginning and the special word [SEP] at the end. This is because the model was pretrained with those, so to get the same results for inference we need to add them as well."
      ],
      "metadata": {
        "id": "cExNmu8pGzEp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Wrapping up, going from sequences to feeding them into model\n",
        "import torch\n",
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
        "\n",
        "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
        "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
        "sequences = [\"I've been waiting for a HuggingFace course my whole life.\", \"So have I!\"]\n",
        "\n",
        "tokens = tokenizer(sequences, padding=True, truncation=True, return_tensors=\"pt\")\n",
        "output = model(**tokens)"
      ],
      "metadata": {
        "id": "JNv9wA8UHAcX"
      },
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Fine Tuning LLMS"
      ],
      "metadata": {
        "id": "cXWcaL44IEtG"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Dataset"
      ],
      "metadata": {
        "id": "7wBd5OWUNDRa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from transformers import AdamW, AutoTokenizer, AutoModelForSequenceClassification\n",
        "\n",
        "# Same as before\n",
        "checkpoint = \"bert-base-uncased\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
        "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
        "sequences = [\n",
        "    \"I've been waiting for a HuggingFace course my whole life.\",\n",
        "    \"This course is amazing!\",\n",
        "]\n",
        "batch = tokenizer(sequences, padding=True, truncation=True, return_tensors=\"pt\")\n",
        "\n",
        "# This is new\n",
        "batch[\"labels\"] = torch.tensor([1, 1])\n",
        "\n",
        "optimizer = AdamW(model.parameters())\n",
        "loss = model(**batch).loss\n",
        "loss.backward()\n",
        "optimizer.step()"
      ],
      "metadata": {
        "id": "-xOVxpuhIINo"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install datasets"
      ],
      "metadata": {
        "id": "M33Zlw7rJWBg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Loading datasets from Huggingface hub\n",
        "from datasets import load_dataset\n",
        "raw_datasets = load_dataset(\"glue\", \"mrpc\")\n",
        "raw_datasets"
      ],
      "metadata": {
        "id": "uoh-WJKXIi7E"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "raw_datasets['train']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VvRgi4nxJ6dY",
        "outputId": "c17ff30f-ce94-42a9-f6eb-23abd9d4cbf5"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['sentence1', 'sentence2', 'label', 'idx'],\n",
              "    num_rows: 3668\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "raw_datasets['train'][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v0uvsWsLKBxj",
        "outputId": "6fc4c940-e1ac-4a2b-b252-452b165c3d1e"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'sentence1': 'Amrozi accused his brother , whom he called \" the witness \" , of deliberately distorting his evidence .',\n",
              " 'sentence2': 'Referring to him as only \" the witness \" , Amrozi accused his brother of deliberately distorting his evidence .',\n",
              " 'label': 1,\n",
              " 'idx': 0}"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "raw_datasets['train'].features"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f8aTer0pKCwJ",
        "outputId": "11949a3f-d2ea-4740-c4ff-13bdb8875695"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'sentence1': Value(dtype='string', id=None),\n",
              " 'sentence2': Value(dtype='string', id=None),\n",
              " 'label': ClassLabel(names=['not_equivalent', 'equivalent'], id=None),\n",
              " 'idx': Value(dtype='int32', id=None)}"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Training"
      ],
      "metadata": {
        "id": "8KupCTNKNHLB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers"
      ],
      "metadata": {
        "id": "Hl_IaJDiqYJk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Restart Runtime after running this cell otherwise you'll encounter an error\n",
        "!pip install accelerate -U"
      ],
      "metadata": {
        "id": "uJIcdnSRNbaJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def compute_metrics(eval_pred):\n",
        "    logits, labels = eval_pred\n",
        "    predictions = np.argmax(logits, axis=-1)\n",
        "    return metric.compute(predictions=predictions, references=labels)"
      ],
      "metadata": {
        "id": "kqbqvFp7Ida6"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import TrainingArguments,Trainer\n",
        "training_args = TrainingArguments(\"test-trainer\", evaluation_strategy=\"epoch\")\n",
        "print(training_args)\n",
        "\n",
        "trainer = Trainer(\n",
        "    model,\n",
        "    training_args,\n",
        "    train_dataset=tokenized_datasets[\"train\"],\n",
        "    eval_dataset=tokenized_datasets[\"validation\"],\n",
        "    data_collator=data_collator,\n",
        "    tokenizer=tokenizer,\n",
        "    compute_metrics=compute_metrics\n",
        ")\n",
        "\n",
        "trainer.train()"
      ],
      "metadata": {
        "id": "f4K0qGTAq0hu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "del model\n",
        "del trainer\n",
        "torch.cuda.empty_cache()"
      ],
      "metadata": {
        "id": "n5HXa61cHTSL"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}