DarijaBERT_Dialect_Identification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "DarijaBERT Dialect Identification.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zYY_H8Kkp8BK"
      },
      "source": [
        "This is a an example notebook describing the different steps to finetune DarijaBERT on Dialect Identification Task.\n",
        " \n",
        "Dataset: [MSDA](https://msda.um6p.ma/msda_datasets) "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ploSy9CKqfiG"
      },
      "source": [
        "# Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1CWzIGZ5RwMB",
        "outputId": "0c2c277c-d745-4ba1-ce66-0544ff175913"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers\n",
            "  Downloading transformers-4.12.2-py3-none-any.whl (3.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.1 MB 8.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.8.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 49.0 MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 57.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.3.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.62.3)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.46-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 63.0 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub>=0.0.17\n",
            "  Downloading huggingface_hub-0.0.19-py3-none-any.whl (56 kB)\n",
            "\u001b[K     |████████████████████████████████| 56 kB 6.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub>=0.0.17->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.6.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.0.19 pyyaml-6.0 sacremoses-0.0.46 tokenizers-0.10.3 transformers-4.12.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s0mQl3uQb8VH",
        "outputId": "1ad3c2a7-0292-4e82-ebb3-884266fbaf44"
      },
      "source": [
        "#options\n",
        "pd.set_option(\"mode.chained_assignment\", None)\n",
        "\n",
        "#Libraries\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import scipy\n",
        "import  os\n",
        "import random\n",
        "import time\n",
        "import datetime\n",
        "import nltk\n",
        "\n",
        "\n",
        "\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "from sklearn.metrics import f1_score, accuracy_score\n",
        "from sklearn.model_selection import train_test_split\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize \n",
        "from sklearn import preprocessing\n",
        "import tensorflow as tf\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "from transformers import get_linear_schedule_with_warmup\n",
        "from transformers import BertTokenizer,AutoTokenizer, AutoModel\n",
        "\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "import itertools\n",
        "import re\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kYbc3xljxeq2",
        "outputId": "c10a8d34-8341-4476-88c3-7792628ce377"
      },
      "source": [
        "!pip install pyarabic"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pyarabic\n",
            "  Downloading PyArabic-0.6.14-py3-none-any.whl (126 kB)\n",
            "\u001b[?25l\r\u001b[K     |██▋                             | 10 kB 34.8 MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 20 kB 29.0 MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 30 kB 18.9 MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 40 kB 15.8 MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 51 kB 8.9 MB/s eta 0:00:01\r\u001b[K     |███████████████▋                | 61 kB 8.6 MB/s eta 0:00:01\r\u001b[K     |██████████████████▏             | 71 kB 8.3 MB/s eta 0:00:01\r\u001b[K     |████████████████████▊           | 81 kB 9.2 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 92 kB 9.6 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 102 kB 8.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 112 kB 8.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 122 kB 8.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 126 kB 8.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: six>=1.14.0 in /usr/local/lib/python3.7/dist-packages (from pyarabic) (1.15.0)\n",
            "Installing collected packages: pyarabic\n",
            "Successfully installed pyarabic-0.6.14\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K0Xe8FsuxVxv"
      },
      "source": [
        "from pyarabic.araby import strip_tatweel,strip_tashkeel"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6OXugcevKtKY"
      },
      "source": [
        "# Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CUMPsJmRrTsr"
      },
      "source": [
        "**Please download the MSDA dataset and then import it like in the next cell**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1GolU0F6thRE"
      },
      "source": [
        "MSDA=pd.read_csv(\"Path/to/MSDA.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "RGZ8ng0bts7V",
        "outputId": "5da4f5cd-01e8-48f8-bb0c-331533c90818"
      },
      "source": [
        "MSDA.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Twits</th>\n",
              "      <th>dialect</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>الدكتورة نجوى قامة في القصف الصاروخي لك ان ترى منشوراتها وتدخلاتها</td>\n",
              "      <td>Algerian</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>سكيكدة_المجاهدة❤ صورة تجمع أحد أكبر و أعظم زعماء ثورتنا المباركة: زيغود يوسف، رويبح حسين، العربي بن مهيدي، عمر أوعمران الذين عقدوا العزم أن تتحرر الجزائر و نعيش اليوم نحن معززين مكرمين تحت راية الشهداء التي إرتضوها أن تكون رمزا لمقاومتهم و تاج فوق رؤوسنا اليوم</td>\n",
              "      <td>Algerian</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>😂😂😂😂😂 شكون يعرفها🤔</td>\n",
              "      <td>Algerian</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>يهود الأندلس للاسف جاو زرعوا الفتنة في وسطنا</td>\n",
              "      <td>Algerian</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>اليهود</td>\n",
              "      <td>Algerian</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                                                                                                                                                                                                                                  Twits   dialect\n",
              "0                                                                                                                                                                                                    الدكتورة نجوى قامة في القصف الصاروخي لك ان ترى منشوراتها وتدخلاتها  Algerian\n",
              "1  سكيكدة_المجاهدة❤ صورة تجمع أحد أكبر و أعظم زعماء ثورتنا المباركة: زيغود يوسف، رويبح حسين، العربي بن مهيدي، عمر أوعمران الذين عقدوا العزم أن تتحرر الجزائر و نعيش اليوم نحن معززين مكرمين تحت راية الشهداء التي إرتضوها أن تكون رمزا لمقاومتهم و تاج فوق رؤوسنا اليوم  Algerian\n",
              "2                                                                                                                                                                                                                                                    😂😂😂😂😂 شكون يعرفها🤔  Algerian\n",
              "3                                                                                                                                                                                                                          يهود الأندلس للاسف جاو زرعوا الفتنة في وسطنا  Algerian\n",
              "4                                                                                                                                                                                                                                                                اليهود  Algerian"
            ]
          },
          "metadata": {},
          "execution_count": 188
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wOLOMl9XtyIH",
        "outputId": "43d29f7f-0729-498d-d4c9-155d6e9e1568"
      },
      "source": [
        "MSDA.dialect.value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Lebanon     14482\n",
              "Algerian    13393\n",
              "Morocco      9965\n",
              "Tunisian     8044\n",
              "egypt        7519\n",
              "Name: dialect, dtype: int64"
            ]
          },
          "execution_count": 50,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5PojpkUdt2xK"
      },
      "source": [
        "#Transform to binary dataset (Moroccan Vs others) and recode to 1/0\n",
        "MSDA['labels']=np.where(MSDA.dialect==\"Morocco\",1,0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "azVuCzWUx611"
      },
      "source": [
        "MSDA.rename(columns={'Twits':'tweet'},inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vaRrIi6wuJ1C"
      },
      "source": [
        "MSDA.drop(columns='dialect',inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gDCnqKlixuu4"
      },
      "source": [
        "## Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PLdNffkl3Izn"
      },
      "source": [
        "\n",
        "def repted(text):\n",
        "   \n",
        "    text=re.sub(r'(.)\\1+', r'\\1', text) # Replace with only one (remove repetitions)  \n",
        "    return text\n",
        "def pre_processing(df,field):\n",
        "    df[field] = df[field].replace(r'http\\S+', 'URL', regex=True).replace(r'www\\S+', 'URL', regex=True)  # Replace URLs with URL string\n",
        "    df[field] = df[field].replace(r'@[^\\s]+', 'USER', regex=True) # Replace user mentions with USER string\n",
        "    df[field] = df[field].replace(r'#[^\\s]+', 'HASHTAG', regex=True) # Replace Hashtags with HASHTAG string\n",
        "\n",
        "    df=df[df[field].apply(lambda x:len(re.findall(r'[\\u0600-\\u06FF]+', x)))>1] #Keep sequences with at least 2 arabic words\n",
        "    df[field] = df[field].apply(strip_tatweel) #Remove Tatweel string \n",
        "    df[field] = df[field].apply(strip_tashkeel) # Remove Diacritics\n",
        "    df[field] = df[field].apply(repted)\n",
        "\n",
        "    return df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dBmZbSTpzFex"
      },
      "source": [
        "MSDA_prc=pre_processing(MSDA,'tweet')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C_Bz7dAn0ZcV"
      },
      "source": [
        "## Train Test split"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iii8TeBI0g2U"
      },
      "source": [
        "#Specify which Data to use\n",
        "Data=MSDA_prc.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pE5iGsuDcuPz"
      },
      "source": [
        "train,valid,train_y,valid_y=train_test_split(Data.tweet.values,Data.labels,test_size=0.2,random_state=42,stratify=Data.labels,)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "omOPIAf3eWhp"
      },
      "source": [
        "dev,test,dev_y,test_y=train_test_split(valid,valid_y,test_size=0.5,random_state=42,stratify=valid_y)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hf5yKySDuAEt"
      },
      "source": [
        "##Variables to be used in the training functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aSYGb-LfgfZt"
      },
      "source": [
        "train_labels=train_y.values\n",
        "train_tweets=train"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ClylLGvaHWOT"
      },
      "source": [
        "dev_labels=dev_y.values\n",
        "dev_tweets=dev"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "61vXYWXHxjxE"
      },
      "source": [
        "test_labels=test_y.values\n",
        "test_tweets=test"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mWsOnYQFuHNy"
      },
      "source": [
        "# GPU device"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GcoVESeVqirY",
        "outputId": "4c492636-23cf-4722-fc7d-7f20d844b0c8"
      },
      "source": [
        "import torch\n",
        "\n",
        "if torch.cuda.is_available():       \n",
        "    device = torch.device(\"cuda\")\n",
        "    print(f'There are {torch.cuda.device_count()} GPU(s) available.')\n",
        "    print('Device name:', torch.cuda.get_device_name(0))\n",
        "\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There are 1 GPU(s) available.\n",
            "Device name: Tesla P100-PCIE-16GB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lXI62U_i9E2V"
      },
      "source": [
        "# Helper Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9cQNvaZ9bnyy"
      },
      "source": [
        "# Function to calculate the accuracy of our predictions vs labels\n",
        "def flat_accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0GEW9HgRTUCA"
      },
      "source": [
        "# Function to calculate the F1-score of our predictions vs labels\n",
        "def flat_f1(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return f1_score(labels_flat,pred_flat) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gpt6tR83keZD"
      },
      "source": [
        "\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    # Round to the nearest second.\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    \n",
        "    # Format as hh:mm:ss\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "795iPKtQAvYi"
      },
      "source": [
        "# Training Functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "60FwRKAoCXLU"
      },
      "source": [
        "### Tokenization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2bBdb3pt8LuQ"
      },
      "source": [
        "#Function doing the Bert tokenizer with pading\n",
        "def Tweets_Tokenizer(tweets,max_len=128):\n",
        "  # Tokenize all of the sentences and map the tokens to their word IDs.\n",
        "  input_ids = []\n",
        "  # Create attention masks\n",
        "  attention_masks = []\n",
        "  MAX_LEN = max_len\n",
        "\n",
        " \n",
        "  # For every sentence...\n",
        "  for tweet in tweets:\n",
        "      \n",
        "      \n",
        "      # `encode` will:\n",
        "      #   (1) Tokenize the sentence.\n",
        "      #   (2) Prepend the `[CLS]` token to the start.\n",
        "      #   (3) Append the `[SEP]` token to the end.\n",
        "      #   (4) Map tokens to their IDs.\n",
        "      encoded_sent = tokenizer.encode(\n",
        "                          tweet,                      # Sentence to encode.\n",
        "                          add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "\n",
        "                          # This function also supports truncation and conversion\n",
        "                          # to pytorch tensors, but we need to do padding, so we\n",
        "                          # can't use these features :( .\n",
        "                          max_length = MAX_LEN,          # Truncate all sentences.\n",
        "                          truncation=True\n",
        "                          #return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                    )\n",
        "      \n",
        "      # Add the encoded sentence to the list.\n",
        "      input_ids.append(encoded_sent)\n",
        "\n",
        "  input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\",value=0, truncating=\"post\", padding=\"post\",)\n",
        "\n",
        "    # For each sentence...\n",
        "  for sent in input_ids: \n",
        "    \n",
        "    \n",
        "    # Create the attention mask.\n",
        "    #   - If a token ID is 0, then it's padding, set the mask to 0.\n",
        "    #   - If a token ID is > 0, then it's a real token, set the mask to 1.\n",
        "    att_mask = [int(token_id > 0) for token_id in sent]\n",
        "        \n",
        "    # Store the attention mask for this sentence.\n",
        "    attention_masks.append(att_mask)\n",
        "    \n",
        "  print('\\n Tokenizer is Done.')\n",
        "\n",
        "  return (input_ids,attention_masks)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wMdsYUIbClSE"
      },
      "source": [
        "### Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cb8gLAaa03Yg"
      },
      "source": [
        "\n",
        "# This training code is based on the `run_glue.py` script here:\n",
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
        "def train_final(data,LR=2e-5,nepochs=3,Batches=16):\n",
        "  # Number of training epochs (authors recommend between 2 and 4)\n",
        "  epochs = nepochs\n",
        "\n",
        "  # Set the seed value all over the place to make this reproducible.\n",
        "  seed_val = 42\n",
        "  batch_size = Batches\n",
        "\n",
        "  random.seed(seed_val)\n",
        "  np.random.seed(seed_val)\n",
        "  torch.manual_seed(seed_val)\n",
        "  torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "\n",
        "  # Use 90% for training and 10% for validation.\n",
        "  input_ids,attention_masks=Tweets_Tokenizer(train_tweets)\n",
        "  # tr_inputs, validation_inputs, tr_labels, validation_labels = train_test_split(input_ids, train_labels, \n",
        "  #                                                           random_state=2018, test_size=0.1)\n",
        "  # # Do the same for the masks.\n",
        "  # train_masks, validation_masks, _, _ = train_test_split(attention_masks, train_labels,\n",
        "  #                                            random_state=2018, test_size=0.1)\n",
        "  \n",
        "  # Convert all inputs and labels into torch tensors, the required datatype \n",
        "  # for our model.\n",
        "  train_inputs = torch.tensor(input_ids)\n",
        "  #validation_inputs = torch.tensor(validation_inputs)\n",
        "\n",
        "  tr_labels = torch.tensor(train_labels)\n",
        "  #validation_labels = torch.tensor(validation_labels)\n",
        "\n",
        "  train_masks = torch.tensor(attention_masks)\n",
        "  #validation_masks = torch.tensor(validation_masks)\n",
        "\n",
        "  # Create the DataLoader for our training set.\n",
        "  train_data = TensorDataset(train_inputs, train_masks, tr_labels)\n",
        "  train_sampler = RandomSampler(train_data)\n",
        "  train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "  # Create the DataLoader for our validation set.\n",
        "  # validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\n",
        "  # validation_sampler = SequentialSampler(validation_data)\n",
        "  # validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)\n",
        "\n",
        "  \n",
        "  # Load BertForSequenceClassification, the pretrained BERT model with a single \n",
        "  # linear classification layer on top. \n",
        "  model = BertForSequenceClassification.from_pretrained(\n",
        "    bert_repo, # Use the 12-layer BERT model, with an uncased vocab.\n",
        "    num_labels = 2, # The number of output labels--2 for binary classification.\n",
        "                    # You can increase this for multi-class tasks.   \n",
        "    output_attentions = False, # Whether the model returns attentions weights.\n",
        "    output_hidden_states = True, # Whether the model returns all hidden-states.\n",
        "    )\n",
        "   \n",
        " \n",
        "\n",
        "  \n",
        "  # Tell pytorch to run this model on the GPU.\n",
        "  model.cuda()\n",
        "  \n",
        "  # Note: AdamW is a class from the huggingface library (as opposed to pytorch) \n",
        "  # I believe the 'W' stands for 'Weight Decay fix\"\n",
        "  optimizer = AdamW(model.parameters(),\n",
        "                  lr = LR, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
        "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
        "                )\n",
        "\n",
        "\n",
        "  # Total number of training steps is number of batches * number of epochs.\n",
        "  total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "  # Create the learning rate scheduler.\n",
        "  scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
        "                                            num_training_steps = total_steps)\n",
        "\n",
        "  # Store the average loss after each epoch so we can plot them.\n",
        "  loss_values = []\n",
        "\n",
        "  # For each epoch...\n",
        "  for epoch_i in range(0, epochs):\n",
        "      \n",
        "      # ========================================\n",
        "      #               Training\n",
        "      # ========================================\n",
        "      \n",
        "      # Perform one full pass over the training set.\n",
        "\n",
        "      print(\"\")\n",
        "      print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "      print('Training...')\n",
        "\n",
        "      # Measure how long the training epoch takes.\n",
        "      t0 = time.time()\n",
        "\n",
        "      # Reset the total loss for this epoch.\n",
        "      total_loss = 0\n",
        "\n",
        "      # Put the model into training mode. Don't be mislead--the call to \n",
        "      # `train` just changes the *mode*, it doesn't *perform* the training.\n",
        "      # `dropout` and `batchnorm` layers behave differently during training\n",
        "      # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
        "      model.train()\n",
        "\n",
        "      # For each batch of training data...\n",
        "      for step, batch in enumerate(train_dataloader):\n",
        "\n",
        "          # Progress update every 40 batches.\n",
        "          if step % 40 == 0 and not step == 0:\n",
        "              # Calculate elapsed time in minutes.\n",
        "              elapsed = format_time(time.time() - t0)\n",
        "              \n",
        "              # Report progress.\n",
        "              print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "          # Unpack this training batch from our dataloader. \n",
        "          #\n",
        "          # As we unpack the batch, we'll also copy each tensor to the GPU using the \n",
        "          # `to` method.\n",
        "          #\n",
        "          # `batch` contains three pytorch tensors:\n",
        "          #   [0]: input ids \n",
        "          #   [1]: attention masks\n",
        "          #   [2]: labels \n",
        "          b_input_ids = batch[0].to(device)\n",
        "          b_input_mask = batch[1].to(device)\n",
        "          b_labels = batch[2].to(device)\n",
        "\n",
        "          # Always clear any previously calculated gradients before performing a\n",
        "          # backward pass. PyTorch doesn't do this automatically because \n",
        "          # accumulating the gradients is \"convenient while training RNNs\". \n",
        "          # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
        "          model.zero_grad()        \n",
        "\n",
        "          # Perform a forward pass (evaluate the model on this training batch).\n",
        "          # This will return the loss (rather than the model output) because we\n",
        "          # have provided the `labels`.\n",
        "          # The documentation for this `model` function is here: \n",
        "          # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "          outputs = model(b_input_ids, \n",
        "                      token_type_ids=None, \n",
        "                      attention_mask=b_input_mask, \n",
        "                      labels=b_labels.to(device=device, dtype=torch.int64))\n",
        "          \n",
        "          # The call to `model` always returns a tuple, so we need to pull the \n",
        "          # loss value out of the tuple.\n",
        "          loss = outputs[0]\n",
        "\n",
        "          # Accumulate the training loss over all of the batches so that we can\n",
        "          # calculate the average loss at the end. `loss` is a Tensor containing a\n",
        "          # single value; the `.item()` function just returns the Python value \n",
        "          # from the tensor.\n",
        "          total_loss += loss.item()\n",
        "\n",
        "          # Perform a backward pass to calculate the gradients.\n",
        "          loss.backward()\n",
        "\n",
        "          # Clip the norm of the gradients to 1.0.\n",
        "          # This is to help prevent the \"exploding gradients\" problem.\n",
        "          torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "          # Update parameters and take a step using the computed gradient.\n",
        "          # The optimizer dictates the \"update rule\"--how the parameters are\n",
        "          # modified based on their gradients, the learning rate, etc.\n",
        "          optimizer.step()\n",
        "\n",
        "          # Update the learning rate.\n",
        "          scheduler.step()\n",
        "\n",
        "      # Calculate the average loss over the training data.\n",
        "      avg_train_loss = total_loss / len(train_dataloader)            \n",
        "      \n",
        "      # Store the loss value for plotting the learning curve.\n",
        "      loss_values.append(avg_train_loss)\n",
        "\n",
        "      print(\"\")\n",
        "      print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "      print(\"  Training epcoh took: {:}\".format(format_time(time.time() - t0)))\n",
        "          \n",
        "    \n",
        "  print(\"\")\n",
        "  print(\"Training complete!\")\n",
        "\n",
        "  return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AQQpmYdPIbCm"
      },
      "source": [
        "### Prediction function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TszHU7vZI3R8"
      },
      "source": [
        "def dev_pred(model,tweets,labels):\n",
        "  dev_inputs, dev_masks = Tweets_Tokenizer(tweets) \n",
        "\n",
        "  # Convert to tensors.\n",
        "  prediction_inputs = torch.tensor(dev_inputs)\n",
        "  prediction_masks = torch.tensor(dev_masks)\n",
        "  prediction_labels = torch.tensor(labels)\n",
        "\n",
        "  # Set the batch size.  \n",
        "  batch_size = 64  \n",
        "\n",
        "  # Create the DataLoader.\n",
        "  prediction_data = TensorDataset(prediction_inputs, prediction_masks, prediction_labels)\n",
        "  prediction_sampler = SequentialSampler(prediction_data)\n",
        "  prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)\n",
        "\n",
        "  # Prediction on test set\n",
        "\n",
        "  print('Predicting labels for {:,} sentences...'.format(len(prediction_inputs)))\n",
        "\n",
        "  # Put model in evaluation mode\n",
        "  model.eval()\n",
        "\n",
        "  # Tracking variables \n",
        "  predictions , true_labels = [], []\n",
        "\n",
        "  # Predict \n",
        "  for batch in prediction_dataloader:\n",
        "    # Add batch to GPU\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    \n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    \n",
        "    # Telling the model not to compute or store gradients, saving memory and \n",
        "    # speeding up prediction\n",
        "    with torch.no_grad():\n",
        "        # Forward pass, calculate logit predictions\n",
        "        outputs = model(b_input_ids, token_type_ids=None, \n",
        "                        attention_mask=b_input_mask)\n",
        "\n",
        "    logits = outputs[0]\n",
        "\n",
        "    # Move logits and labels to CPU\n",
        "    logits = logits.detach().cpu().numpy()\n",
        "    label_ids = b_labels.to('cpu').numpy()\n",
        "    \n",
        "    # Store predictions and true labels\n",
        "    predictions.append(logits)\n",
        "    true_labels.append(label_ids)\n",
        "\n",
        "  \n",
        "\n",
        "  # Combine the predictions for each batch into a single list of 0s and 1s.\n",
        "  flat_predictions = [item for sublist in predictions for item in sublist]\n",
        "  Last_predictions = np.argmax(flat_predictions, axis=1).flatten()\n",
        "\n",
        "  # Combine the correct labels for each batch into a single list.\n",
        "  flat_true_labels = [item for sublist in true_labels for item in sublist]\n",
        "\n",
        "  \n",
        "  indiv_F1=f1_score(flat_true_labels, Last_predictions) * 100\n",
        "  macro_F1=f1_score(flat_true_labels, Last_predictions,average='macro')*100\n",
        "\n",
        "  \n",
        "  print(\"\\nF1 Score on positive: %.2f\" %indiv_F1, \"%\") \n",
        "  #print(indiv_F1) \n",
        "\n",
        "  return (flat_predictions,Last_predictions,flat_true_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zHDzj2B6_5ZR"
      },
      "source": [
        "# Running Models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ohei9AHSo7C0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "509592e1-aae8-4c9e-8f3d-fbcd37519ff2"
      },
      "source": [
        "\n",
        "#https://github.com/alisafaya/Arabic-BERT\n",
        "tokenizer_DarijaBert = AutoTokenizer.from_pretrained(\"Kamel/DarijaBERT\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n",
            "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n",
            "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QGTcqW4xozOF"
      },
      "source": [
        "tokenizer=tokenizer_DarijaBert\n",
        "bert_repo=\"Kamel/DarijaBERT\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pwSa_NB7k0SV",
        "outputId": "afb7f26b-8a71-427f-9edc-e94db63c0602"
      },
      "source": [
        "model_final=train_final(train,Batches=64,nepochs=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " Tokenizer is Done.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n",
            "Some weights of the model checkpoint at /content/drive/MyDrive/Doctorat/DarijaBERT/Models/DarijaBert_v1.0/epoch40 were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at /content/drive/MyDrive/Doctorat/DarijaBERT/Models/DarijaBert_v1.0/epoch40 and are newly initialized: ['classifier.weight', 'bert.pooler.dense.weight', 'bert.pooler.dense.bias', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "======== Epoch 1 / 1 ========\n",
            "Training...\n",
            "  Batch    40  of    569.    Elapsed: 0:00:29.\n",
            "  Batch    80  of    569.    Elapsed: 0:00:58.\n",
            "  Batch   120  of    569.    Elapsed: 0:01:27.\n",
            "  Batch   160  of    569.    Elapsed: 0:01:56.\n",
            "  Batch   200  of    569.    Elapsed: 0:02:25.\n",
            "  Batch   240  of    569.    Elapsed: 0:02:54.\n",
            "  Batch   280  of    569.    Elapsed: 0:03:23.\n",
            "  Batch   320  of    569.    Elapsed: 0:03:51.\n",
            "  Batch   360  of    569.    Elapsed: 0:04:20.\n",
            "  Batch   400  of    569.    Elapsed: 0:04:49.\n",
            "  Batch   440  of    569.    Elapsed: 0:05:18.\n",
            "  Batch   480  of    569.    Elapsed: 0:05:47.\n",
            "  Batch   520  of    569.    Elapsed: 0:06:16.\n",
            "  Batch   560  of    569.    Elapsed: 0:06:45.\n",
            "\n",
            "  Average training loss: 0.17\n",
            "  Training epcoh took: 0:06:51\n",
            "\n",
            "Training complete!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "784KOiml2nVc"
      },
      "source": [
        "# Prediction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yMG8_D1j_c6G"
      },
      "source": [
        "flat_predictions,Last_predictions,flat_true_labels=dev_pred(model_final,dev_tweets,dev_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o6juic7pJAwR"
      },
      "source": [
        "flat_predictions,Last_predictions,flat_true_labels=dev_pred(model_final,test_tweets,test_labels)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}