Slot-filling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "# Experiments on Cross Lingual Transfer for Slot-Filling\n",
    "## Santichai Pornavalai\n",
    "## 10.2.2021\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook is used to run the experiments for slot-flling cross-lingual experiments using XLM-R.The blocks for training and testing are meant to be run individually and correspond to the experiments listed in the paper. Models are saved under /model directory with the prefix slot followed by the language tags. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from simpletransformers.ner import NERModel\n",
    "import pandas as pd\n",
    "import logging\n",
    "import sklearn\n",
    "import sklearn_crfsuite\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn_crfsuite import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7f6d41b4b190>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(1366)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path to files\n",
    "path_2_en_train = \"data/formatted/en_format_train.conll\"\n",
    "path_2_en_test = \"data/formatted/en_format_test.conll\"\n",
    "path_2_es_train = \"data/formatted/es_format_train.conll\"\n",
    "path_2_es_test = \"data/formatted/es_format_test.conll\"\n",
    "path_2_th_train = \"data/formatted/th_format_train.conll\"\n",
    "path_2_th_test = \"data/formatted/th_format_test.conll\"\n",
    "\n",
    "path_2_checkpoints = \"prelim_models\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = ['B-alarm/alarm_modifier', 'I-reminder/reference', 'B-reminder/reminder_modifier', 'I-reminder/todo', 'NoLabel',\n",
    "          'B-timer/attributes', 'B-datetime', 'B-reminder/todo', 'B-reminder/recurring_period', 'B-timer/noun', 'I-weather/noun',\n",
    "          'B-negation', 'B-reminder/noun', 'I-weather/attribute', 'I-alarm/alarm_modifier', 'B-weather/noun', 'I-datetime', 'B-weather/attribute',\n",
    "          'I-reminder/recurring_period', 'I-location', 'B-demonstrative_reference', 'B-location', 'I-reminder/reminder_modifier', 'B-reminder/reference',\n",
    "          'B-weather/temperatureUnit', 'I-reminder/noun', 'B-news/type', 'I-demonstrative_reference', 'I-negation', 'B-alarm/recurring_period', \"I-alarm/recurring_period\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = {'fp16': True,\n",
    "        'reprocess_input_data': True,\n",
    "        'evaluate_during_training':False,\n",
    "        \"evaluate_during_training_verbose\": False,\n",
    "        'overwrite_output_dir': True,\n",
    "        'num_train_epochs': 3,# set to 1 for test.\n",
    "        'save_steps':-1,\n",
    "        \"save_model_every_epoch\":False, }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "macro = lambda x,y:  metrics.flat_f1_score(x,y, average= 'macro')\n",
    "micro = lambda x,y:  metrics.flat_f1_score(x,y, average= 'micro')\n",
    "report = lambda x,y:  metrics.flat_classification_report(x,y,digits = 5)\n",
    "report_dict = lambda x,y:  metrics.flat_classification_report(x,y,digits = 5,output_dict = True,labels = list(range(len(labels))),target_names = labels)\n",
    "accuracy = lambda x,y:  metrics.flat_accuracy_score(x,y)\n",
    "seq_accuracy = lambda x,y:  metrics.sequence_accuracy_score(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_model(model,test_file, return_dict = False):\n",
    "    result, _, predictions = model.eval_model\\\n",
    "                                        (test_file,\n",
    "                                         macro=macro,\n",
    "                                         micro=micro,\n",
    "                                         accuracy=accuracy,\n",
    "                                         report=report,\n",
    "                                         seq_accuracy = seq_accuracy)\n",
    "    print(\"tested on: \",test_file)\n",
    "    print(result[\"report\"])\n",
    "    print(\"sequence accuracy\", result[\"seq_accuracy\"])\n",
    "    if return_dict:\n",
    "        return result\n",
    "    \n",
    "    \n",
    "def load_test(model_path, test_file, model_type = 'xlmroberta'):\n",
    "    model = NERModel(\n",
    "        'xlmroberta', model_path\n",
    "    )\n",
    "    test_model(model, test_file)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at xlm-roberta-base were not used when initializing XLMRobertaForTokenClassification: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
      "- This IS expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of XLMRobertaForTokenClassification were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d5ed2e5a8f442ab9621cb2888829413",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=30521.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d45d021134b648c7a07229a139c870e4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=3.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03781214664f4f67857acefe0b5a14eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 0 of 3', max=3816.0, style=ProgressStyle(de…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/santi/anaconda3/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:136: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c788645a2eb4e7fa185bfa073e97a80",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 1 of 3', max=3816.0, style=ProgressStyle(de…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c71db623b50747d3af622841d382e8e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 2 of 3', max=3816.0, style=ProgressStyle(de…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce048b4f402d4c20b203611415f0b34f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=8621.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c17fb36feae49688acad76c012e8b1d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=1078.0, style=ProgressStyle(desc…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/santi/anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/home/santi/anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tested on:  data/formatted/en_format_test.conll\n",
      "                              precision    recall  f1-score   support\n",
      "\n",
      "      B-alarm/alarm_modifier    1.00000   1.00000   1.00000         3\n",
      "                  B-datetime    0.97635   0.98538   0.98085      6158\n",
      "                  B-location    0.97795   0.98544   0.98168      1305\n",
      "                  B-negation    0.00000   0.00000   0.00000         3\n",
      "                 B-news/type    0.00000   0.00000   0.00000         1\n",
      "             B-reminder/noun    0.98017   0.95816   0.96904       980\n",
      " B-reminder/recurring_period    0.88462   0.86250   0.87342        80\n",
      "        B-reminder/reference    0.84615   0.77647   0.80982        85\n",
      "B-reminder/reminder_modifier    0.00000   0.00000   0.00000         2\n",
      "             B-reminder/todo    0.94313   0.94891   0.94601      1468\n",
      "         B-weather/attribute    0.98512   0.97389   0.97947      2719\n",
      "              B-weather/noun    0.99429   0.98553   0.98989      1589\n",
      "   B-weather/temperatureUnit    0.99664   1.00000   0.99832       297\n",
      "                  I-datetime    0.96785   0.98583   0.97676      8184\n",
      "                  I-location    0.97132   0.97692   0.97411       520\n",
      "             I-reminder/noun    0.00000   0.00000   0.00000        13\n",
      " I-reminder/recurring_period    0.85294   0.78378   0.81690        74\n",
      "             I-reminder/todo    0.97206   0.96299   0.96750      3540\n",
      "         I-weather/attribute    0.91304   0.78750   0.84564       160\n",
      "              I-weather/noun    0.92593   0.96899   0.94697       129\n",
      "                     NoLabel    0.98912   0.98682   0.98797     35281\n",
      "\n",
      "                    accuracy                        0.98169     62591\n",
      "                   macro avg    0.77032   0.75853   0.76402     62591\n",
      "                weighted avg    0.98140   0.98169   0.98151     62591\n",
      "\n",
      "sequence accuracy 0.9315624637513049\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1394324274a4512b94653c8b94aba59",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=3043.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a30460dac5e479e841b29c13019f4bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=381.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tested on:  data/formatted/es_format_test.conll\n",
      "                             precision    recall  f1-score   support\n",
      "\n",
      "     B-alarm/alarm_modifier    0.00000   0.00000   0.00000         4\n",
      "                 B-datetime    0.90274   0.89573   0.89922      2062\n",
      "                 B-location    0.76299   0.88346   0.81882       266\n",
      "                 B-negation    0.00000   0.00000   0.00000         2\n",
      "            B-reminder/noun    0.96933   0.96146   0.96538       493\n",
      "B-reminder/recurring_period    0.61290   0.73077   0.66667        26\n",
      "       B-reminder/reference    0.85714   0.42857   0.57143        14\n",
      "            B-reminder/todo    0.90073   0.84290   0.87085       732\n",
      "        B-weather/attribute    0.87215   0.80140   0.83528       715\n",
      "             B-weather/noun    0.90173   0.80412   0.85014       388\n",
      "     I-alarm/alarm_modifier    0.00000   0.00000   0.00000         4\n",
      "                 I-datetime    0.97292   0.91056   0.94071      4774\n",
      "                 I-location    0.84298   0.71831   0.77567       142\n",
      "                 I-negation    0.00000   0.00000   0.00000         1\n",
      "            I-reminder/noun    0.00000   0.00000   0.00000        21\n",
      "I-reminder/recurring_period    0.70588   0.40678   0.51613        59\n",
      "       I-reminder/reference    0.00000   0.00000   0.00000         3\n",
      "            I-reminder/todo    0.95000   0.90244   0.92561      2337\n",
      "        I-weather/attribute    0.68966   0.15385   0.25157       130\n",
      "             I-weather/noun    0.50000   0.01818   0.03509        55\n",
      "                    NoLabel    0.91422   0.98072   0.94630     10791\n",
      "\n",
      "                   accuracy                        0.92398     23019\n",
      "                  macro avg    0.58835   0.49711   0.51756     23019\n",
      "               weighted avg    0.92149   0.92398   0.92005     23019\n",
      "\n",
      "sequence accuracy 0.751889582648702\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a13464b6812240bf9a8e281280797f3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1692.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db0755b878c34ae18f43fd1b1d360f84",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=212.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tested on:  data/formatted/th_format_test.conll\n",
      "                             precision    recall  f1-score   support\n",
      "\n",
      "                 B-datetime    0.63767   0.56431   0.59875      1104\n",
      "                 B-location    0.93388   0.70186   0.80142       161\n",
      "            B-reminder/noun    0.00000   0.00000   0.00000       228\n",
      "B-reminder/recurring_period    0.80000   0.40000   0.53333        10\n",
      "       B-reminder/reference    0.00000   0.00000   0.00000         7\n",
      "            B-reminder/todo    0.79468   0.68750   0.73721       304\n",
      "        B-weather/attribute    0.96296   0.17219   0.29213       453\n",
      "             B-weather/noun    0.66667   0.01835   0.03571       218\n",
      "                 I-datetime    0.82816   0.77460   0.80048      2134\n",
      "                 I-location    0.85714   0.82979   0.84324       188\n",
      "            I-reminder/noun    0.00000   0.00000   0.00000       450\n",
      "I-reminder/recurring_period    0.94118   0.64000   0.76190        25\n",
      "       I-reminder/reference    0.00000   0.00000   0.00000        11\n",
      "            I-reminder/todo    0.78030   0.95011   0.85687       942\n",
      "        I-weather/attribute    0.94340   0.44053   0.60060       227\n",
      "             I-weather/noun    0.00000   0.00000   0.00000       146\n",
      "                    NoLabel    0.75071   0.93217   0.83166      7062\n",
      "\n",
      "                   accuracy                        0.76328     13670\n",
      "                  macro avg    0.58216   0.41832   0.45255     13670\n",
      "               weighted avg    0.72335   0.76328   0.72144     13670\n",
      "\n",
      "sequence accuracy 0.28841607565011823\n"
     ]
    }
   ],
   "source": [
    "\n",
    "args[\"output_dir\"] = \"models/slot_en_train\"\n",
    "model = NERModel('xlmroberta','xlm-roberta-base', labels = labels,  args=args)\n",
    "#train eng\n",
    "model.train_model( path_2_en_train)\n",
    "#baseline\n",
    "test_model(model, path_2_en_test)\n",
    "#zero-shot cross_lingual\"\n",
    "test_model(model, path_2_es_test)\n",
    "test_model(model, path_2_th_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at xlm-roberta-base were not used when initializing XLMRobertaForTokenClassification: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
      "- This IS expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of XLMRobertaForTokenClassification were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b6196b46e6942669e398ba7d0253c9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=3617.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "551b56f09e00405ca552be091c0b11fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=3.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1fdc668bc3e14c8f9750423a812d310e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 0 of 3', max=453.0, style=ProgressStyle(des…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "803ab639489c4df7b04277bbf009c528",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 1 of 3', max=453.0, style=ProgressStyle(des…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f3cdf12b22a4601a5e988ae504de188",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 2 of 3', max=453.0, style=ProgressStyle(des…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5891532d3c6747f8a00858a9d17f596e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=8621.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a06e69b026e34ecdbdb42a0e6e266535",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=1078.0, style=ProgressStyle(desc…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tested on:  data/formatted/en_format_test.conll\n",
      "                              precision    recall  f1-score   support\n",
      "\n",
      "      B-alarm/alarm_modifier    0.00000   0.00000   0.00000         3\n",
      "                  B-datetime    0.92963   0.91604   0.92279      6158\n",
      "                  B-location    0.88880   0.83295   0.85997      1305\n",
      "                  B-negation    0.00000   0.00000   0.00000         3\n",
      "                 B-news/type    0.00000   0.00000   0.00000         1\n",
      "             B-reminder/noun    0.98152   0.92143   0.95053       980\n",
      " B-reminder/recurring_period    1.00000   0.13750   0.24176        80\n",
      "        B-reminder/reference    0.85185   0.27059   0.41071        85\n",
      "B-reminder/reminder_modifier    0.00000   0.00000   0.00000         2\n",
      "             B-reminder/todo    0.90649   0.81880   0.86042      1468\n",
      "         B-weather/attribute    0.79468   0.95513   0.86755      2719\n",
      "              B-weather/noun    0.91020   0.89301   0.90152      1589\n",
      "   B-weather/temperatureUnit    0.00000   0.00000   0.00000       297\n",
      "                  I-datetime    0.91243   0.97141   0.94100      8184\n",
      "                  I-location    0.76117   0.95000   0.84517       520\n",
      "             I-reminder/noun    0.00000   0.00000   0.00000        13\n",
      " I-reminder/recurring_period    1.00000   0.09459   0.17284        74\n",
      "             I-reminder/todo    0.93804   0.89379   0.91538      3540\n",
      "         I-weather/attribute    0.06849   0.15625   0.09524       160\n",
      "              I-weather/noun    0.00000   0.00000   0.00000       129\n",
      "                     NoLabel    0.96716   0.96171   0.96443     35281\n",
      "\n",
      "                    accuracy                        0.93389     62591\n",
      "                   macro avg    0.56716   0.46539   0.47378     62591\n",
      "                weighted avg    0.93189   0.93389   0.93112     62591\n",
      "\n",
      "sequence accuracy 0.7321656420368867\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d697f475f1114ce39b28e14bce915bfc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=3043.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8edae93e23d141a0b326602fc7e76880",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=381.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tested on:  data/formatted/es_format_test.conll\n",
      "                             precision    recall  f1-score   support\n",
      "\n",
      "     B-alarm/alarm_modifier    0.00000   0.00000   0.00000         4\n",
      "                 B-datetime    0.94413   0.95878   0.95140      2062\n",
      "                 B-location    0.86617   0.87594   0.87103       266\n",
      "                 B-negation    0.00000   0.00000   0.00000         2\n",
      "            B-reminder/noun    0.96579   0.97363   0.96970       493\n",
      "B-reminder/recurring_period    0.77778   0.26923   0.40000        26\n",
      "       B-reminder/reference    0.87500   0.50000   0.63636        14\n",
      "            B-reminder/todo    0.84239   0.84699   0.84469       732\n",
      "        B-weather/attribute    0.93352   0.92308   0.92827       715\n",
      "             B-weather/noun    0.93532   0.96907   0.95190       388\n",
      "     I-alarm/alarm_modifier    0.00000   0.00000   0.00000         4\n",
      "                 I-datetime    0.95044   0.98010   0.96504      4774\n",
      "                 I-location    0.89922   0.81690   0.85609       142\n",
      "                 I-negation    0.00000   0.00000   0.00000         1\n",
      "            I-reminder/noun    0.00000   0.00000   0.00000        21\n",
      "I-reminder/recurring_period    0.61111   0.18644   0.28571        59\n",
      "       I-reminder/reference    0.00000   0.00000   0.00000         3\n",
      "            I-reminder/todo    0.93565   0.95807   0.94672      2337\n",
      "        I-weather/attribute    0.77119   0.70000   0.73387       130\n",
      "             I-weather/noun    0.76000   0.34545   0.47500        55\n",
      "                    NoLabel    0.97615   0.96710   0.97160     10791\n",
      "\n",
      "                   accuracy                        0.95360     23019\n",
      "                  macro avg    0.62114   0.53670   0.56130     23019\n",
      "               weighted avg    0.95122   0.95360   0.95166     23019\n",
      "\n",
      "sequence accuracy 0.8399605652316793\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca2a93b7df2b4d87bda0545e39d248bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1692.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34168757e70c47d2ace604d37c674094",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=212.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tested on:  data/formatted/th_format_test.conll\n",
      "                             precision    recall  f1-score   support\n",
      "\n",
      "                 B-datetime    0.62903   0.60054   0.61446      1104\n",
      "                 B-location    0.91275   0.84472   0.87742       161\n",
      "            B-reminder/noun    0.00000   0.00000   0.00000       228\n",
      "B-reminder/recurring_period    1.00000   0.10000   0.18182        10\n",
      "       B-reminder/reference    0.00000   0.00000   0.00000         7\n",
      "            B-reminder/todo    0.68621   0.65461   0.67003       304\n",
      "        B-weather/attribute    0.81423   0.45475   0.58357       453\n",
      "             B-weather/noun    0.35762   0.24771   0.29268       218\n",
      "                 I-datetime    0.82008   0.95689   0.88322      2134\n",
      "                 I-location    0.82828   0.87234   0.84974       188\n",
      "            I-reminder/noun    0.00000   0.00000   0.00000       450\n",
      "I-reminder/recurring_period    1.00000   0.08000   0.14815        25\n",
      "       I-reminder/reference    0.00000   0.00000   0.00000        11\n",
      "            I-reminder/todo    0.67152   0.78344   0.72317       942\n",
      "        I-weather/attribute    0.71287   0.63436   0.67133       227\n",
      "             I-weather/noun    1.00000   0.01370   0.02703       146\n",
      "                    NoLabel    0.82080   0.90413   0.86045      7062\n",
      "\n",
      "                   accuracy                        0.78537     13670\n",
      "                  macro avg    0.60314   0.42042   0.43430     13670\n",
      "               weighted avg    0.74429   0.78537   0.75462     13670\n",
      "\n",
      "sequence accuracy 0.3421985815602837\n"
     ]
    }
   ],
   "source": [
    "args[\"output_dir\"] = \"models/slot_es_train\"\n",
    "model = NERModel('xlmroberta','xlm-roberta-base', labels = labels,  args=args)\n",
    "#train es\n",
    "model.train_model(path_2_es_train)\n",
    "\n",
    "test_model(model, path_2_en_test)\n",
    "test_model(model, path_2_es_test)\n",
    "test_model(model, path_2_th_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at xlm-roberta-base were not used when initializing XLMRobertaForTokenClassification: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
      "- This IS expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of XLMRobertaForTokenClassification were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c14b4cc25da4948b8323a403ec81a13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2156.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c8d1d0f3984443b18d7ff5aedea2aedc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=3.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7777e0d8e194910b7e376496cbef3ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 0 of 3', max=270.0, style=ProgressStyle(des…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe140bb546ce4659ac0b05c9fc3ae692",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 1 of 3', max=270.0, style=ProgressStyle(des…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f238486e84f4490098eea599531f915a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 2 of 3', max=270.0, style=ProgressStyle(des…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "76caa6e4a1894ef98fd1e95029b8bfa5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=8621.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c165ecd39e9a47a683b5f902a33780b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=1078.0, style=ProgressStyle(desc…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tested on:  data/formatted/en_format_test.conll\n",
      "                              precision    recall  f1-score   support\n",
      "\n",
      "      B-alarm/alarm_modifier    0.00000   0.00000   0.00000         3\n",
      "                  B-datetime    0.56345   0.54726   0.55524      6158\n",
      "                  B-location    0.83890   0.77011   0.80304      1305\n",
      "                  B-negation    0.00000   0.00000   0.00000         3\n",
      "                 B-news/type    0.00000   0.00000   0.00000         1\n",
      "             B-reminder/noun    0.00000   0.00000   0.00000       980\n",
      " B-reminder/recurring_period    0.00000   0.00000   0.00000        80\n",
      "        B-reminder/reference    0.00000   0.00000   0.00000        85\n",
      "B-reminder/reminder_modifier    0.00000   0.00000   0.00000         2\n",
      "             B-reminder/todo    0.87240   0.42847   0.57469      1468\n",
      "         B-weather/attribute    0.60395   0.70835   0.65200      2719\n",
      "              B-weather/noun    0.46705   0.41032   0.43685      1589\n",
      "   B-weather/temperatureUnit    0.00000   0.00000   0.00000       297\n",
      "                  I-datetime    0.82237   0.71163   0.76300      8184\n",
      "                  I-location    0.67479   0.90577   0.77340       520\n",
      "             I-reminder/noun    0.00000   0.00000   0.00000        13\n",
      " I-reminder/recurring_period    0.00000   0.00000   0.00000        74\n",
      "             I-reminder/todo    0.90953   0.63898   0.75062      3540\n",
      "         I-weather/attribute    0.10206   0.83750   0.18194       160\n",
      "              I-weather/noun    0.13988   0.36434   0.20215       129\n",
      "                     NoLabel    0.83323   0.90193   0.86622     35281\n",
      "\n",
      "                    accuracy                        0.76914     62591\n",
      "                   macro avg    0.32512   0.34403   0.31234     62591\n",
      "                weighted avg    0.76627   0.76914   0.76205     62591\n",
      "\n",
      "sequence accuracy 0.26679039554576034\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4168f9a4ea334244b84e67650c23c561",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=3043.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bbcc7fdf2ca14820b6852686bd946869",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=381.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tested on:  data/formatted/es_format_test.conll\n",
      "                             precision    recall  f1-score   support\n",
      "\n",
      "     B-alarm/alarm_modifier    0.00000   0.00000   0.00000         4\n",
      "                 B-datetime    0.49274   0.54316   0.51672      2062\n",
      "                 B-location    0.79348   0.82331   0.80812       266\n",
      "                 B-negation    0.00000   0.00000   0.00000         2\n",
      "            B-reminder/noun    0.00000   0.00000   0.00000       493\n",
      "B-reminder/recurring_period    0.00000   0.00000   0.00000        26\n",
      "       B-reminder/reference    0.00000   0.00000   0.00000        14\n",
      "            B-reminder/todo    0.78846   0.16803   0.27703       732\n",
      "        B-weather/attribute    0.55359   0.65734   0.60102       715\n",
      "             B-weather/noun    0.49144   0.51804   0.50439       388\n",
      "     I-alarm/alarm_modifier    0.00000   0.00000   0.00000         4\n",
      "                 I-datetime    0.92314   0.76477   0.83652      4774\n",
      "                 I-location    0.73780   0.85211   0.79085       142\n",
      "                 I-negation    0.00000   0.00000   0.00000         1\n",
      "            I-reminder/noun    0.00000   0.00000   0.00000        21\n",
      "I-reminder/recurring_period    0.00000   0.00000   0.00000        59\n",
      "       I-reminder/reference    0.00000   0.00000   0.00000         3\n",
      "            I-reminder/todo    0.86710   0.34061   0.48909      2337\n",
      "        I-weather/attribute    0.24029   0.76154   0.36531       130\n",
      "             I-weather/noun    0.16379   0.34545   0.22222        55\n",
      "                    NoLabel    0.73608   0.92012   0.81787     10791\n",
      "\n",
      "                   accuracy                        0.72757     23019\n",
      "                  macro avg    0.32323   0.31879   0.29663     23019\n",
      "               weighted avg    0.73471   0.72757   0.70563     23019\n",
      "\n",
      "sequence accuracy 0.22346368715083798\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7101de21e6746a3bb059fd8ad0e11c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1692.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e8d3c95e18a84485950beefa234212de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=212.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tested on:  data/formatted/th_format_test.conll\n",
      "                             precision    recall  f1-score   support\n",
      "\n",
      "                 B-datetime    0.95957   0.96739   0.96346      1104\n",
      "                 B-location    0.95679   0.96273   0.95975       161\n",
      "            B-reminder/noun    0.96087   0.96930   0.96507       228\n",
      "B-reminder/recurring_period    0.00000   0.00000   0.00000        10\n",
      "       B-reminder/reference    0.00000   0.00000   0.00000         7\n",
      "            B-reminder/todo    0.90429   0.90132   0.90280       304\n",
      "        B-weather/attribute    0.98043   0.99558   0.98795       453\n",
      "             B-weather/noun    0.97273   0.98165   0.97717       218\n",
      "                 I-datetime    0.97147   0.98922   0.98026      2134\n",
      "                 I-location    0.94444   0.90426   0.92391       188\n",
      "            I-reminder/noun    0.95879   0.98222   0.97036       450\n",
      "I-reminder/recurring_period    0.00000   0.00000   0.00000        25\n",
      "       I-reminder/reference    0.00000   0.00000   0.00000        11\n",
      "            I-reminder/todo    0.96750   0.94798   0.95764       942\n",
      "        I-weather/attribute    0.92083   0.97357   0.94647       227\n",
      "             I-weather/noun    0.99296   0.96575   0.97917       146\n",
      "                    NoLabel    0.98443   0.98457   0.98450      7062\n",
      "\n",
      "                   accuracy                        0.97396     13670\n",
      "                  macro avg    0.73383   0.73680   0.73521     13670\n",
      "               weighted avg    0.97023   0.97396   0.97205     13670\n",
      "\n",
      "sequence accuracy 0.9018912529550828\n"
     ]
    }
   ],
   "source": [
    "args[\"output_dir\"] = \"models/slot_th_train\"\n",
    "\n",
    "model = NERModel('xlmroberta','xlm-roberta-base', labels = labels,  args=args)\n",
    "#thai th\n",
    "model.train_model(path_2_th_train)\n",
    "\n",
    "\n",
    "test_model(model, path_2_en_test)\n",
    "test_model(model, path_2_es_test)\n",
    "test_model(model, path_2_th_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at xlm-roberta-base were not used when initializing XLMRobertaForTokenClassification: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
      "- This IS expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of XLMRobertaForTokenClassification were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce18b728571745fea2df3d47880d984d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=30521.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33b579d50d1d4c6aa3c3de14837db08b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=3.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb81d200db6e4835aeed3f423e416ee4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 0 of 3', max=3816.0, style=ProgressStyle(de…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24ab0b6f92684160a68fa04d1ae0f6c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 1 of 3', max=3816.0, style=ProgressStyle(de…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73313a48cf984390b7f8360b6c709a52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 2 of 3', max=3816.0, style=ProgressStyle(de…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da39afd7797443dca76875a34f121375",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2156.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eee6809ff85c4fe7b3ff450afa12f1cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=3.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4f4be0dd25844a9a9ff5b133f821e660",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 0 of 3', max=270.0, style=ProgressStyle(des…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a7b19c1ae754da08d9bb056f70f68d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 1 of 3', max=270.0, style=ProgressStyle(des…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ec2c9e6ba56423196e248f6e3077b34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 2 of 3', max=270.0, style=ProgressStyle(des…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e1365ac0edd4fe89f866a8ee5e5c6b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=8621.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6f25faa24824e9c9c7aa75d6563ea4e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=1078.0, style=ProgressStyle(desc…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tested on:  data/formatted/en_format_test.conll\n",
      "                              precision    recall  f1-score   support\n",
      "\n",
      "      B-alarm/alarm_modifier    0.00000   0.00000   0.00000         3\n",
      "                  B-datetime    0.96502   0.96314   0.96408      6158\n",
      "                  B-location    0.97057   0.98544   0.97795      1305\n",
      "                  B-negation    0.00000   0.00000   0.00000         3\n",
      "                 B-news/type    0.00000   0.00000   0.00000         1\n",
      "             B-reminder/noun    0.90734   0.95918   0.93254       980\n",
      " B-reminder/recurring_period    0.71579   0.85000   0.77714        80\n",
      "        B-reminder/reference    0.81579   0.72941   0.77019        85\n",
      "B-reminder/reminder_modifier    0.00000   0.00000   0.00000         2\n",
      "             B-reminder/todo    0.93467   0.95504   0.94474      1468\n",
      "         B-weather/attribute    0.98068   0.97058   0.97560      2719\n",
      "              B-weather/noun    0.98357   0.97923   0.98139      1589\n",
      "   B-weather/temperatureUnit    1.00000   0.99663   0.99831       297\n",
      "                  I-datetime    0.95184   0.98045   0.96593      8184\n",
      "                  I-location    0.95167   0.98462   0.96786       520\n",
      "             I-reminder/noun    0.00000   0.00000   0.00000        13\n",
      " I-reminder/recurring_period    0.70909   0.52703   0.60465        74\n",
      "             I-reminder/todo    0.96624   0.96215   0.96419      3540\n",
      "         I-weather/attribute    0.79042   0.82500   0.80734       160\n",
      "              I-weather/noun    0.86207   0.96899   0.91241       129\n",
      "                     NoLabel    0.98759   0.97928   0.98342     35281\n",
      "\n",
      "                    accuracy                        0.97407     62591\n",
      "                   macro avg    0.69011   0.69601   0.69180     62591\n",
      "                weighted avg    0.97398   0.97407   0.97393     62591\n",
      "\n",
      "sequence accuracy 0.899663612109964\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4ecdda4e76c47d3b37db84bcd2268d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=3043.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4e7ce3e8dfc4e82b75fbbe276b4710e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=381.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tested on:  data/formatted/es_format_test.conll\n",
      "                             precision    recall  f1-score   support\n",
      "\n",
      "     B-alarm/alarm_modifier    0.00000   0.00000   0.00000         4\n",
      "                 B-datetime    0.70312   0.69835   0.70073      2062\n",
      "                 B-location    0.82007   0.89098   0.85405       266\n",
      "                 B-negation    0.00000   0.00000   0.00000         2\n",
      "            B-reminder/noun    0.94653   0.96957   0.95792       493\n",
      "B-reminder/recurring_period    0.55556   0.57692   0.56604        26\n",
      "       B-reminder/reference    0.55556   0.35714   0.43478        14\n",
      "            B-reminder/todo    0.86517   0.73634   0.79557       732\n",
      "        B-weather/attribute    0.88679   0.85455   0.87037       715\n",
      "             B-weather/noun    0.90857   0.81959   0.86179       388\n",
      "     I-alarm/alarm_modifier    0.00000   0.00000   0.00000         4\n",
      "                 I-datetime    0.96509   0.81064   0.88115      4774\n",
      "                 I-location    0.85612   0.83803   0.84698       142\n",
      "                 I-negation    0.00000   0.00000   0.00000         1\n",
      "            I-reminder/noun    0.00000   0.00000   0.00000        21\n",
      "I-reminder/recurring_period    0.67742   0.35593   0.46667        59\n",
      "       I-reminder/reference    0.00000   0.00000   0.00000         3\n",
      "            I-reminder/todo    0.96823   0.82157   0.88889      2337\n",
      "        I-weather/attribute    0.59649   0.52308   0.55738       130\n",
      "             I-weather/noun    0.17647   0.05455   0.08333        55\n",
      "                    NoLabel    0.86705   0.97906   0.91966     10791\n",
      "\n",
      "                   accuracy                        0.87793     23019\n",
      "                  macro avg    0.54039   0.48982   0.50882     23019\n",
      "               weighted avg    0.87979   0.87793   0.87480     23019\n",
      "\n",
      "sequence accuracy 0.6368715083798883\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "602650573b0147b0bc3357813b9fa394",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1692.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "acfe575d7c2b4d3fa5f9acaa93810fa1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=212.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tested on:  data/formatted/th_format_test.conll\n",
      "                             precision    recall  f1-score   support\n",
      "\n",
      "                 B-datetime    0.95217   0.95562   0.95389      1104\n",
      "                 B-location    0.93491   0.98137   0.95758       161\n",
      "            B-reminder/noun    0.96087   0.96930   0.96507       228\n",
      "B-reminder/recurring_period    0.69231   0.90000   0.78261        10\n",
      "       B-reminder/reference    1.00000   0.28571   0.44444         7\n",
      "            B-reminder/todo    0.91089   0.90789   0.90939       304\n",
      "        B-weather/attribute    0.98444   0.97792   0.98117       453\n",
      "             B-weather/noun    0.96818   0.97706   0.97260       218\n",
      "                 I-datetime    0.98496   0.98172   0.98334      2134\n",
      "                 I-location    0.95676   0.94149   0.94906       188\n",
      "            I-reminder/noun    0.95680   0.98444   0.97043       450\n",
      "I-reminder/recurring_period    0.94737   0.72000   0.81818        25\n",
      "       I-reminder/reference    1.00000   0.18182   0.30769        11\n",
      "            I-reminder/todo    0.97461   0.93737   0.95563       942\n",
      "        I-weather/attribute    0.92672   0.94714   0.93682       227\n",
      "             I-weather/noun    0.98592   0.95890   0.97222       146\n",
      "                    NoLabel    0.98197   0.98711   0.98453      7062\n",
      "\n",
      "                   accuracy                        0.97447     13670\n",
      "                  macro avg    0.94817   0.85852   0.87321     13670\n",
      "               weighted avg    0.97460   0.97447   0.97412     13670\n",
      "\n",
      "sequence accuracy 0.8983451536643026\n"
     ]
    }
   ],
   "source": [
    "args[\"output_dir\"] = \"models/slot_en_th_train\"\n",
    "\n",
    "model = NERModel('xlmroberta','xlm-roberta-base', labels = labels,  args=args)\n",
    "\n",
    "#sequentially train\n",
    "model.train_model(path_2_en_train)\n",
    "model.train_model(path_2_th_train)\n",
    "\n",
    "\n",
    "test_model(model, path_2_en_test)\n",
    "test_model(model, path_2_es_test)\n",
    "test_model(model, path_2_th_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at xlm-roberta-base were not used when initializing XLMRobertaForTokenClassification: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
      "- This IS expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of XLMRobertaForTokenClassification were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e1ed2f17172b44df8d35e1003a12370a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=30521.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7a7d2c9330e48778495fba0936b71aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=3.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1240c36aa8ad4b799e87a38c7a3e0533",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 0 of 3', max=3816.0, style=ProgressStyle(de…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e600e896a0646eb9913ae63f49a3829",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 1 of 3', max=3816.0, style=ProgressStyle(de…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df21097367114218b2c12c4b796066f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 2 of 3', max=3816.0, style=ProgressStyle(de…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d46369fff95b4d088d65136d81bdad1f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=3617.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f698d4582f5c44dd98c936f80ea459ee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=3.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59fa12c0ba4e40a5aa4a425720f3b143",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 0 of 3', max=453.0, style=ProgressStyle(des…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af43c4c0c1b34ac99636584666c61daa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 1 of 3', max=453.0, style=ProgressStyle(des…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51b73c7c5600492989fa85decb2ed70e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 2 of 3', max=453.0, style=ProgressStyle(des…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13056e7ec64748429620de143ce305cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=8621.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ad86dd160b847a5b09249e0e45043ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=1078.0, style=ProgressStyle(desc…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tested on:  data/formatted/en_format_test.conll\n",
      "                              precision    recall  f1-score   support\n",
      "\n",
      "      B-alarm/alarm_modifier    0.60000   1.00000   0.75000         3\n",
      "                  B-datetime    0.96815   0.98230   0.97517      6158\n",
      "                  B-location    0.97428   0.98697   0.98059      1305\n",
      "                  B-negation    0.00000   0.00000   0.00000         3\n",
      "                 B-news/type    0.00000   0.00000   0.00000         1\n",
      "             B-reminder/noun    0.97609   0.95816   0.96704       980\n",
      " B-reminder/recurring_period    0.89855   0.77500   0.83221        80\n",
      "        B-reminder/reference    0.92982   0.62353   0.74648        85\n",
      "B-reminder/reminder_modifier    0.00000   0.00000   0.00000         2\n",
      "             B-reminder/todo    0.92580   0.90940   0.91753      1468\n",
      "         B-weather/attribute    0.95874   0.97426   0.96644      2719\n",
      "              B-weather/noun    0.98558   0.98930   0.98744      1589\n",
      "   B-weather/temperatureUnit    1.00000   0.99663   0.99831       297\n",
      "                  I-datetime    0.94763   0.98839   0.96758      8184\n",
      "                  I-location    0.94590   0.97500   0.96023       520\n",
      "             I-reminder/noun    0.00000   0.00000   0.00000        13\n",
      " I-reminder/recurring_period    0.91935   0.77027   0.83824        74\n",
      "             I-reminder/todo    0.94087   0.96638   0.95346      3540\n",
      "         I-weather/attribute    0.75275   0.85625   0.80117       160\n",
      "              I-weather/noun    0.91176   0.96124   0.93585       129\n",
      "                     NoLabel    0.99102   0.97594   0.98342     35281\n",
      "\n",
      "                    accuracy                        0.97479     62591\n",
      "                   macro avg    0.74411   0.74710   0.74101     62591\n",
      "                weighted avg    0.97490   0.97479   0.97466     62591\n",
      "\n",
      "sequence accuracy 0.9038394617793759\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64862e93481c41c9a17d4868139a919e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=3043.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "415433f886c64b839d09b32df9d99767",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=381.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tested on:  data/formatted/es_format_test.conll\n",
      "                             precision    recall  f1-score   support\n",
      "\n",
      "     B-alarm/alarm_modifier    0.00000   0.00000   0.00000         4\n",
      "                 B-datetime    0.94503   0.95878   0.95185      2062\n",
      "                 B-location    0.88764   0.89098   0.88931       266\n",
      "                 B-negation    0.00000   0.00000   0.00000         2\n",
      "            B-reminder/noun    0.96000   0.97363   0.96677       493\n",
      "B-reminder/recurring_period    0.67647   0.88462   0.76667        26\n",
      "       B-reminder/reference    0.85714   0.42857   0.57143        14\n",
      "            B-reminder/todo    0.84146   0.84836   0.84490       732\n",
      "        B-weather/attribute    0.94714   0.92727   0.93710       715\n",
      "             B-weather/noun    0.95153   0.96134   0.95641       388\n",
      "     I-alarm/alarm_modifier    0.00000   0.00000   0.00000         4\n",
      "                 I-datetime    0.96142   0.97612   0.96871      4774\n",
      "                 I-location    0.92969   0.83803   0.88148       142\n",
      "                 I-negation    0.00000   0.00000   0.00000         1\n",
      "            I-reminder/noun    0.00000   0.00000   0.00000        21\n",
      "I-reminder/recurring_period    0.73469   0.61017   0.66667        59\n",
      "       I-reminder/reference    0.00000   0.00000   0.00000         3\n",
      "            I-reminder/todo    0.94108   0.96363   0.95222      2337\n",
      "        I-weather/attribute    0.81197   0.73077   0.76923       130\n",
      "             I-weather/noun    0.77778   0.50909   0.61538        55\n",
      "                    NoLabel    0.97537   0.96877   0.97206     10791\n",
      "\n",
      "                   accuracy                        0.95677     23019\n",
      "                  macro avg    0.62850   0.59382   0.60525     23019\n",
      "               weighted avg    0.95521   0.95677   0.95579     23019\n",
      "\n",
      "sequence accuracy 0.8504765034505423\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "76a464a76a214db6a38308e29be8b6db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1692.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf9a01cf9c484b15b05a9d5001da443c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=212.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/santi/anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n",
      "/home/santi/anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tested on:  data/formatted/th_format_test.conll\n",
      "                             precision    recall  f1-score   support\n",
      "\n",
      "     B-alarm/alarm_modifier    0.00000   0.00000   0.00000         0\n",
      "                 B-datetime    0.61117   0.51540   0.55921      1104\n",
      "                 B-location    0.89928   0.77640   0.83333       161\n",
      "            B-reminder/noun    0.00000   0.00000   0.00000       228\n",
      "B-reminder/recurring_period    0.25000   0.40000   0.30769        10\n",
      "       B-reminder/reference    0.00000   0.00000   0.00000         7\n",
      "            B-reminder/todo    0.82500   0.75987   0.79110       304\n",
      "        B-weather/attribute    0.98010   0.43488   0.60245       453\n",
      "             B-weather/noun    1.00000   0.16972   0.29020       218\n",
      "                 I-datetime    0.82969   0.83552   0.83259      2134\n",
      "                 I-location    0.89119   0.91489   0.90289       188\n",
      "            I-reminder/noun    0.00000   0.00000   0.00000       450\n",
      "I-reminder/recurring_period    0.21296   0.92000   0.34586        25\n",
      "       I-reminder/reference    0.00000   0.00000   0.00000        11\n",
      "            I-reminder/todo    0.80282   0.96815   0.87777       942\n",
      "        I-weather/attribute    0.88701   0.69163   0.77723       227\n",
      "             I-weather/noun    0.96552   0.19178   0.32000       146\n",
      "                    NoLabel    0.79108   0.92651   0.85345      7062\n",
      "\n",
      "                   accuracy                        0.78866     13670\n",
      "                  macro avg    0.55255   0.47249   0.46077     13670\n",
      "               weighted avg    0.75811   0.78866   0.75812     13670\n",
      "\n",
      "sequence accuracy 0.3433806146572104\n"
     ]
    }
   ],
   "source": [
    "args[\"output_dir\"] = \"models/slot_en_es_train\"\n",
    "model = NERModel('xlmroberta','xlm-roberta-base', labels = labels,  args=args)\n",
    "\n",
    "model.train_model(path_2_en_train )\n",
    "model.train_model(path_2_es_train )\n",
    "\n",
    "\n",
    "test_model(model, path_2_en_test)\n",
    "test_model(model, path_2_es_test)\n",
    "test_model(model, path_2_th_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.train_model(path_2_th_train,output_dir = path_2_checkpoints)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "    \n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}