Named Entity Recognition.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "696d3bf3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/envs/transformers-1/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import transformers\n",
    "\n",
    "#Set to avoid warning messages.\n",
    "transformers.logging.set_verbosity_error()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "413f0377",
   "metadata": {},
   "source": [
    "## 1. Running the standard NER Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8d4f73e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: 100%|██████████████████████████████████████████████████████████████████████████████| 998/998 [00:00<00:00, 155kB/s]\n",
      "Downloading: 100%|█████████████████████████████████████████████████████████████████████████| 1.33G/1.33G [00:43<00:00, 30.4MB/s]\n",
      "Downloading: 100%|███████████████████████████████████████████████████████████████████████████| 60.0/60.0 [00:00<00:00, 35.1kB/s]\n",
      "Downloading: 100%|████████████████████████████████████████████████████████████████████████████| 213k/213k [00:00<00:00, 810kB/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'entity': 'I-PER',\n",
       "  'score': 0.99887806,\n",
       "  'index': 1,\n",
       "  'word': 'Sam',\n",
       "  'start': 0,\n",
       "  'end': 3},\n",
       " {'entity': 'I-LOC',\n",
       "  'score': 0.99972683,\n",
       "  'index': 4,\n",
       "  'word': 'California',\n",
       "  'start': 12,\n",
       "  'end': 22},\n",
       " {'entity': 'I-ORG',\n",
       "  'score': 0.9960085,\n",
       "  'index': 15,\n",
       "  'word': 'Google',\n",
       "  'start': 64,\n",
       "  'end': 70},\n",
       " {'entity': 'I-PER',\n",
       "  'score': 0.99891376,\n",
       "  'index': 18,\n",
       "  'word': 'John',\n",
       "  'start': 89,\n",
       "  'end': 93},\n",
       " {'entity': 'I-PER',\n",
       "  'score': 0.99921584,\n",
       "  'index': 19,\n",
       "  'word': 'Smith',\n",
       "  'start': 94,\n",
       "  'end': 99}]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "input_text=\"Sam went to California on the 23rd of August. \\\n",
    "There, he visited Google headquarters with John Smith and bought a cap for $23\"\n",
    "\n",
    "basic_ner = pipeline(\"ner\")\n",
    "\n",
    "basic_ner(input_text)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6267ddcd",
   "metadata": {},
   "source": [
    "## 2. Understanding the model architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b10debb2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BertForTokenClassification(\n",
      "  (bert): BertModel(\n",
      "    (embeddings): BertEmbeddings(\n",
      "      (word_embeddings): Embedding(28996, 1024, padding_idx=0)\n",
      "      (position_embeddings): Embedding(512, 1024)\n",
      "      (token_type_embeddings): Embedding(2, 1024)\n",
      "      (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "      (dropout): Dropout(p=0.1, inplace=False)\n",
      "    )\n",
      "    (encoder): BertEncoder(\n",
      "      (layer): ModuleList(\n",
      "        (0): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (1): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (2): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (3): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (4): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (5): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (6): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (7): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (8): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (9): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (10): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (11): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (12): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (13): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (14): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (15): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (16): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (17): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (18): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (19): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (20): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (21): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (22): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (23): BertLayer(\n",
      "          (attention): BertAttention(\n",
      "            (self): BertSelfAttention(\n",
      "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): BertSelfOutput(\n",
      "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): BertIntermediate(\n",
      "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): BertOutput(\n",
      "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      "  (classifier): Linear(in_features=1024, out_features=9, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "#Print model architecture\n",
    "print(basic_ner.model)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "774664f5",
   "metadata": {},
   "source": [
    "## 3. Reviewing model configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ccd0c0e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BertConfig {\n",
      "  \"_name_or_path\": \"dbmdz/bert-large-cased-finetuned-conll03-english\",\n",
      "  \"_num_labels\": 9,\n",
      "  \"architectures\": [\n",
      "    \"BertForTokenClassification\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 1024,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"O\",\n",
      "    \"1\": \"B-MISC\",\n",
      "    \"2\": \"I-MISC\",\n",
      "    \"3\": \"B-PER\",\n",
      "    \"4\": \"I-PER\",\n",
      "    \"5\": \"B-ORG\",\n",
      "    \"6\": \"I-ORG\",\n",
      "    \"7\": \"B-LOC\",\n",
      "    \"8\": \"I-LOC\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 4096,\n",
      "  \"label2id\": {\n",
      "    \"B-LOC\": 7,\n",
      "    \"B-MISC\": 1,\n",
      "    \"B-ORG\": 5,\n",
      "    \"B-PER\": 3,\n",
      "    \"I-LOC\": 8,\n",
      "    \"I-MISC\": 2,\n",
      "    \"I-ORG\": 6,\n",
      "    \"I-PER\": 4,\n",
      "    \"O\": 0\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 16,\n",
      "  \"num_hidden_layers\": 24,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.25.1\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(basic_ner.model.config)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ce6950f",
   "metadata": {},
   "source": [
    "## 4. Using a Custom Model and tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0e9219da",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: 100%|██████████████████████████████████████████████████████████████████████████████| 423/423 [00:00<00:00, 251kB/s]\n",
      "Downloading: 100%|██████████████████████████████████████████████████████████████████████████████| 970/970 [00:00<00:00, 607kB/s]\n",
      "Downloading: 100%|███████████████████████████████████████████████████████████████████████████| 811k/811k [00:00<00:00, 5.26MB/s]\n",
      "Downloading: 100%|██████████████████████████████████████████████████████████████████████████████| 210/210 [00:00<00:00, 112kB/s]\n",
      "Downloading: 100%|███████████████████████████████████████████████████████████████████████████| 440M/440M [00:15<00:00, 29.1MB/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 'O', 1: 'I-LOC', 2: 'I-PER', 3: 'I-MISC', 4: 'I-ORG', 5: 'I-DATE'}\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, TFAutoModelForTokenClassification\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"Jean-Baptiste/camembert-ner-with-dates\", \n",
    "                                          from_pt=True)\n",
    "\n",
    "model = TFAutoModelForTokenClassification.from_pretrained(\"Jean-Baptiste/camembert-ner-with-dates\",\n",
    "                                                          from_pt=True)\n",
    "\n",
    "print(model.config.id2label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bf11cdfe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'entity_group': 'PER',\n",
       "  'score': 0.9776213,\n",
       "  'word': 'Sam',\n",
       "  'start': 0,\n",
       "  'end': 3},\n",
       " {'entity_group': 'LOC',\n",
       "  'score': 0.9936407,\n",
       "  'word': 'California',\n",
       "  'start': 11,\n",
       "  'end': 22},\n",
       " {'entity_group': 'DATE',\n",
       "  'score': 0.92355955,\n",
       "  'word': 'August',\n",
       "  'start': 37,\n",
       "  'end': 44},\n",
       " {'entity_group': 'ORG',\n",
       "  'score': 0.5721681,\n",
       "  'word': 'Google',\n",
       "  'start': 63,\n",
       "  'end': 70},\n",
       " {'entity_group': 'PER',\n",
       "  'score': 0.9938346,\n",
       "  'word': 'John Smith',\n",
       "  'start': 88,\n",
       "  'end': 99},\n",
       " {'entity_group': 'DATE',\n",
       "  'score': 0.64064246,\n",
       "  'word': '23',\n",
       "  'start': 122,\n",
       "  'end': 124}]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Prediction\n",
    "enhanced_ner = pipeline('ner', \n",
    "                        model=model, \n",
    "                        tokenizer=tokenizer, \n",
    "                        aggregation_strategy=\"simple\")\n",
    "enhanced_ner(input_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8caf3b89",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4 (v3.11.4:d2340ef257, Jun  6 2023, 19:15:51) [Clang 13.0.0 (clang-1300.0.29.30)]"
  },
  "vscode": {
   "interpreter": {
    "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}