SkimLit_Project_V2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f8af3164",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import models, layers\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b1b6f0b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = '../SkimLit_Project/pubmed-rct-master/PubMed_20k_RCT_numbers_replaced_with_at_sign/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7f22233b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../SkimLit_Project/pubmed-rct-master/PubMed_20k_RCT_numbers_replaced_with_at_sign/dev.txt',\n",
       " '../SkimLit_Project/pubmed-rct-master/PubMed_20k_RCT_numbers_replaced_with_at_sign/test.txt',\n",
       " '../SkimLit_Project/pubmed-rct-master/PubMed_20k_RCT_numbers_replaced_with_at_sign/train.txt']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "filenames = [data_dir + filename for filename in os.listdir(data_dir)]\n",
    "filenames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "554893eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_lines(filename):\n",
    "    with open(filename, 'r') as f:\n",
    "        return f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "68f45ffb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['###24293578\\n',\n",
       " 'OBJECTIVE\\tTo investigate the efficacy of @ weeks of daily low-dose oral prednisolone in improving pain , mobility , and systemic low-grade inflammation in the short term and whether the effect would be sustained at @ weeks in older adults with moderate to severe knee osteoarthritis ( OA ) .\\n',\n",
       " 'METHODS\\tA total of @ patients with primary knee OA were randomized @:@ ; @ received @ mg/day of prednisolone and @ received placebo for @ weeks .\\n',\n",
       " 'METHODS\\tOutcome measures included pain reduction and improvement in function scores and systemic inflammation markers .\\n',\n",
       " 'METHODS\\tPain was assessed using the visual analog pain scale ( @-@ mm ) .\\n',\n",
       " 'METHODS\\tSecondary outcome measures included the Western Ontario and McMaster Universities Osteoarthritis Index scores , patient global assessment ( PGA ) of the severity of knee OA , and @-min walk distance ( @MWD ) .\\n',\n",
       " 'METHODS\\tSerum levels of interleukin @ ( IL-@ ) , IL-@ , tumor necrosis factor ( TNF ) - , and high-sensitivity C-reactive protein ( hsCRP ) were measured .\\n',\n",
       " 'RESULTS\\tThere was a clinically relevant reduction in the intervention group compared to the placebo group for knee pain , physical function , PGA , and @MWD at @ weeks .\\n',\n",
       " 'RESULTS\\tThe mean difference between treatment arms ( @ % CI ) was @ ( @-@ @ ) , p < @ ; @ ( @-@ @ ) , p < @ ; @ ( @-@ @ ) , p < @ ; and @ ( @-@ @ ) , p < @ , respectively .\\n',\n",
       " 'RESULTS\\tFurther , there was a clinically relevant reduction in the serum levels of IL-@ , IL-@ , TNF - , and hsCRP at @ weeks in the intervention group when compared to the placebo group .\\n',\n",
       " 'RESULTS\\tThese differences remained significant at @ weeks .\\n',\n",
       " 'RESULTS\\tThe Outcome Measures in Rheumatology Clinical Trials-Osteoarthritis Research Society International responder rate was @ % in the intervention group and @ % in the placebo group ( p < @ ) .\\n',\n",
       " 'CONCLUSIONS\\tLow-dose oral prednisolone had both a short-term and a longer sustained effect resulting in less knee pain , better physical function , and attenuation of systemic inflammation in older patients with knee OA ( ClinicalTrials.gov identifier NCT@ ) .\\n',\n",
       " '\\n',\n",
       " '###24854809\\n',\n",
       " 'BACKGROUND\\tEmotional eating is associated with overeating and the development of obesity .\\n',\n",
       " 'BACKGROUND\\tYet , empirical evidence for individual ( trait ) differences in emotional eating and cognitive mechanisms that contribute to eating during sad mood remain equivocal .\\n',\n",
       " 'OBJECTIVE\\tThe aim of this study was to test if attention bias for food moderates the effect of self-reported emotional eating during sad mood ( vs neutral mood ) on actual food intake .\\n',\n",
       " 'OBJECTIVE\\tIt was expected that emotional eating is predictive of elevated attention for food and higher food intake after an experimentally induced sad mood and that attentional maintenance on food predicts food intake during a sad versus a neutral mood .\\n',\n",
       " 'METHODS\\tParticipants ( N = @ ) were randomly assigned to one of the two experimental mood induction conditions ( sad/neutral ) .\\n']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_lines = get_lines(data_dir + 'train.txt')\n",
    "train_lines[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ff67ae73",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['###24293578\\n']\n",
      "['OBJECTIVE', 'To investigate the efficacy of @ weeks of daily low-dose oral prednisolone in improving pain , mobility , and systemic low-grade inflammation in the short term and whether the effect would be sustained at @ weeks in older adults with moderate to severe knee osteoarthritis ( OA ) .\\n']\n",
      "['METHODS', 'A total of @ patients with primary knee OA were randomized @:@ ; @ received @ mg/day of prednisolone and @ received placebo for @ weeks .\\n']\n",
      "['METHODS', 'Outcome measures included pain reduction and improvement in function scores and systemic inflammation markers .\\n']\n",
      "['METHODS', 'Pain was assessed using the visual analog pain scale ( @-@ mm ) .\\n']\n",
      "['METHODS', 'Secondary outcome measures included the Western Ontario and McMaster Universities Osteoarthritis Index scores , patient global assessment ( PGA ) of the severity of knee OA , and @-min walk distance ( @MWD ) .\\n']\n",
      "['METHODS', 'Serum levels of interleukin @ ( IL-@ ) , IL-@ , tumor necrosis factor ( TNF ) - , and high-sensitivity C-reactive protein ( hsCRP ) were measured .\\n']\n",
      "['RESULTS', 'There was a clinically relevant reduction in the intervention group compared to the placebo group for knee pain , physical function , PGA , and @MWD at @ weeks .\\n']\n",
      "['RESULTS', 'The mean difference between treatment arms ( @ % CI ) was @ ( @-@ @ ) , p < @ ; @ ( @-@ @ ) , p < @ ; @ ( @-@ @ ) , p < @ ; and @ ( @-@ @ ) , p < @ , respectively .\\n']\n",
      "['RESULTS', 'Further , there was a clinically relevant reduction in the serum levels of IL-@ , IL-@ , TNF - , and hsCRP at @ weeks in the intervention group when compared to the placebo group .\\n']\n"
     ]
    }
   ],
   "source": [
    "for line in train_lines[:10]:\n",
    "    print(line.split('\\t'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9829581b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_text_with_line_numbers(filename):\n",
    "    input_lines = get_lines(filename)\n",
    "    abstract_lines = \"\"\n",
    "    abstract_samples = []\n",
    "    \n",
    "    for line in input_lines:\n",
    "        if line.startswith(\"###\"):\n",
    "            abstract_id = line\n",
    "            abstract_lines= \"\"\n",
    "        elif line.isspace():\n",
    "            abstract_line_split = abstract_lines.splitlines()\n",
    "            \n",
    "            for abstract_line_number, abstract_line in enumerate(abstract_line_split):\n",
    "                line_data = {}\n",
    "                target_text_split = abstract_line.split('\\t')\n",
    "                line_data['target'] = target_text_split[0]\n",
    "                line_data['text'] = target_text_split[1].lower()\n",
    "                line_data['line_number'] = abstract_line_number\n",
    "                line_data['total_lines'] = len(abstract_line_split)-1\n",
    "                \n",
    "                abstract_samples.append(line_data)\n",
    "        else:\n",
    "            abstract_lines += line\n",
    "            \n",
    "    return abstract_samples     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e9b25c52",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 672 ms\n",
      "Wall time: 698 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_samples = preprocess_text_with_line_numbers(data_dir + 'train.txt')\n",
    "val_samples = preprocess_text_with_line_numbers(data_dir + 'dev.txt')\n",
    "test_samples = preprocess_text_with_line_numbers(data_dir + 'test.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8e314fc4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'target': 'OBJECTIVE',\n",
       "  'text': 'to investigate the efficacy of @ weeks of daily low-dose oral prednisolone in improving pain , mobility , and systemic low-grade inflammation in the short term and whether the effect would be sustained at @ weeks in older adults with moderate to severe knee osteoarthritis ( oa ) .',\n",
       "  'line_number': 0,\n",
       "  'total_lines': 11},\n",
       " {'target': 'METHODS',\n",
       "  'text': 'a total of @ patients with primary knee oa were randomized @:@ ; @ received @ mg/day of prednisolone and @ received placebo for @ weeks .',\n",
       "  'line_number': 1,\n",
       "  'total_lines': 11},\n",
       " {'target': 'METHODS',\n",
       "  'text': 'outcome measures included pain reduction and improvement in function scores and systemic inflammation markers .',\n",
       "  'line_number': 2,\n",
       "  'total_lines': 11},\n",
       " {'target': 'METHODS',\n",
       "  'text': 'pain was assessed using the visual analog pain scale ( @-@ mm ) .',\n",
       "  'line_number': 3,\n",
       "  'total_lines': 11},\n",
       " {'target': 'METHODS',\n",
       "  'text': 'secondary outcome measures included the western ontario and mcmaster universities osteoarthritis index scores , patient global assessment ( pga ) of the severity of knee oa , and @-min walk distance ( @mwd ) .',\n",
       "  'line_number': 4,\n",
       "  'total_lines': 11},\n",
       " {'target': 'METHODS',\n",
       "  'text': 'serum levels of interleukin @ ( il-@ ) , il-@ , tumor necrosis factor ( tnf ) - , and high-sensitivity c-reactive protein ( hscrp ) were measured .',\n",
       "  'line_number': 5,\n",
       "  'total_lines': 11},\n",
       " {'target': 'RESULTS',\n",
       "  'text': 'there was a clinically relevant reduction in the intervention group compared to the placebo group for knee pain , physical function , pga , and @mwd at @ weeks .',\n",
       "  'line_number': 6,\n",
       "  'total_lines': 11},\n",
       " {'target': 'RESULTS',\n",
       "  'text': 'the mean difference between treatment arms ( @ % ci ) was @ ( @-@ @ ) , p < @ ; @ ( @-@ @ ) , p < @ ; @ ( @-@ @ ) , p < @ ; and @ ( @-@ @ ) , p < @ , respectively .',\n",
       "  'line_number': 7,\n",
       "  'total_lines': 11},\n",
       " {'target': 'RESULTS',\n",
       "  'text': 'further , there was a clinically relevant reduction in the serum levels of il-@ , il-@ , tnf - , and hscrp at @ weeks in the intervention group when compared to the placebo group .',\n",
       "  'line_number': 8,\n",
       "  'total_lines': 11},\n",
       " {'target': 'RESULTS',\n",
       "  'text': 'these differences remained significant at @ weeks .',\n",
       "  'line_number': 9,\n",
       "  'total_lines': 11},\n",
       " {'target': 'RESULTS',\n",
       "  'text': 'the outcome measures in rheumatology clinical trials-osteoarthritis research society international responder rate was @ % in the intervention group and @ % in the placebo group ( p < @ ) .',\n",
       "  'line_number': 10,\n",
       "  'total_lines': 11},\n",
       " {'target': 'CONCLUSIONS',\n",
       "  'text': 'low-dose oral prednisolone had both a short-term and a longer sustained effect resulting in less knee pain , better physical function , and attenuation of systemic inflammation in older patients with knee oa ( clinicaltrials.gov identifier nct@ ) .',\n",
       "  'line_number': 11,\n",
       "  'total_lines': 11},\n",
       " {'target': 'BACKGROUND',\n",
       "  'text': 'emotional eating is associated with overeating and the development of obesity .',\n",
       "  'line_number': 0,\n",
       "  'total_lines': 10},\n",
       " {'target': 'BACKGROUND',\n",
       "  'text': 'yet , empirical evidence for individual ( trait ) differences in emotional eating and cognitive mechanisms that contribute to eating during sad mood remain equivocal .',\n",
       "  'line_number': 1,\n",
       "  'total_lines': 10}]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_samples[:14]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9eefe554",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>text</th>\n",
       "      <th>line_number</th>\n",
       "      <th>total_lines</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>OBJECTIVE</td>\n",
       "      <td>to investigate the efficacy of @ weeks of dail...</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>METHODS</td>\n",
       "      <td>a total of @ patients with primary knee oa wer...</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>METHODS</td>\n",
       "      <td>outcome measures included pain reduction and i...</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>METHODS</td>\n",
       "      <td>pain was assessed using the visual analog pain...</td>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>METHODS</td>\n",
       "      <td>secondary outcome measures included the wester...</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      target                                               text  line_number  \\\n",
       "0  OBJECTIVE  to investigate the efficacy of @ weeks of dail...            0   \n",
       "1    METHODS  a total of @ patients with primary knee oa wer...            1   \n",
       "2    METHODS  outcome measures included pain reduction and i...            2   \n",
       "3    METHODS  pain was assessed using the visual analog pain...            3   \n",
       "4    METHODS  secondary outcome measures included the wester...            4   \n",
       "\n",
       "   total_lines  \n",
       "0           11  \n",
       "1           11  \n",
       "2           11  \n",
       "3           11  \n",
       "4           11  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.DataFrame(train_samples)\n",
    "val_df = pd.DataFrame(val_samples)\n",
    "test_df = pd.DataFrame(test_samples)\n",
    "\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ecd1399b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:ylabel='Frequency'>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk0AAAGeCAYAAACJuDVEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA1b0lEQVR4nO3df3QU9b3/8deaH9skTbYhIVn2GjFXQy4xaGvoDQErKJCABPxxT8GbukLFgCdKTEkOSvuHtNcbfhrsvTlFbD2glpr+QKw9QBqsNG0K4UfaVEOR2ookSJagLBtIwyYm8/3D63zdBGGIwd2kz8c5c447n/fOvGfOePLis7OzNsMwDAEAAOCirgp2AwAAAEMBoQkAAMACQhMAAIAFhCYAAAALCE0AAAAWEJoAAAAsIDQBAABYQGgCAACwgNAEAABgQXiwGxhOent7deLECcXGxspmswW7HQAAYIFhGDp79qxcLpeuuuoi80lGEI0ePdqQ1G8pKioyDMMwent7jSeeeMIYNWqU8YUvfMGYPHmy0dTUFLCN8+fPG4888oiRkJBgREdHG7NnzzZaWloCak6fPm3cd999RlxcnBEXF2fcd999htfrDag5duyYkZ+fb0RHRxsJCQnGkiVLDL/ff1nH09LScsHjYWFhYWFhYQn9pW9+6CuoM00HDhxQT0+P+bqpqUnTp0/X17/+dUnSmjVrVFFRoc2bN2vMmDF68sknNX36dB05ckSxsbGSpJKSEv3qV79SVVWVEhISVFpaqvz8fDU0NCgsLEySVFBQoOPHj6u6ulqStGjRIrndbv3qV7+SJPX09GjWrFkaOXKk6urq9MEHH2j+/PkyDEP/+7//a/l4Pu6ppaVFcXFxn/0EAQCAK669vV0pKSnm3/FPdVlTKVfYo48+alx33XVGb2+v0dvbazidTmPVqlXm+Pnz5w2Hw2E888wzhmEYxpkzZ4yIiAijqqrKrHnvvfeMq666yqiurjYMwzD+8pe/GJKM+vp6s2bv3r2GJOOtt94yDMMwduzYYVx11VXGe++9Z9a89NJLht1uN3w+n+X+fT6fIemy3gMAAILL6t/vkLkRvKurSz/+8Y/1wAMPyGaz6ejRo/J4PMrNzTVr7Ha7Jk+erD179kiSGhoa1N3dHVDjcrmUmZlp1uzdu1cOh0PZ2dlmzYQJE+RwOAJqMjMz5XK5zJq8vDz5/X41NDR8as9+v1/t7e0BCwAAGJ5CJjS98sorOnPmjBYsWCBJ8ng8kqTk5OSAuuTkZHPM4/EoMjJS8fHxF61JSkrqt7+kpKSAmr77iY+PV2RkpFlzIStXrpTD4TCXlJSUyzhiAAAwlIRMaHruuec0c+bMgNkeSf2+hWYYxiW/mda35kL1A6npa/ny5fL5fObS0tJy0b4AAMDQFRKh6dixY3rttdf04IMPmuucTqck9ZvpaWtrM2eFnE6nurq65PV6L1pz8uTJfvs8depUQE3f/Xi9XnV3d/ebgfoku92uuLi4gAUAAAxPIRGaNm3apKSkJM2aNctcl5qaKqfTqV27dpnrurq6VFtbq4kTJ0qSsrKyFBEREVDT2tqqpqYmsyYnJ0c+n0/79+83a/bt2yefzxdQ09TUpNbWVrOmpqZGdrtdWVlZV+agAQDAkBL0h1v29vZq06ZNmj9/vsLD/387NptNJSUlKi8vV1pamtLS0lReXq7o6GgVFBRIkhwOhxYuXKjS0lIlJCRoxIgRKisr07hx4zRt2jRJ0tixYzVjxgwVFhZq48aNkj565EB+fr7S09MlSbm5ucrIyJDb7dbatWt1+vRplZWVqbCwkNkjAAAgKQRC02uvvabm5mY98MAD/caWLVumzs5OFRUVyev1Kjs7WzU1NQHPUVi/fr3Cw8M1d+5cdXZ2aurUqdq8ebP5jCZJ2rJli4qLi81v2c2ZM0eVlZXmeFhYmLZv366ioiJNmjRJUVFRKigo0Lp1667gkQMAgKHEZhiGEewmhov29nY5HA75fD5mqAAAGCKs/v0OiXuaAAAAQh2hCQAAwAJCEwAAgAWEJgAAAAsITQAAABYQmgAAACwI+nOagFBy7ePbg93CZXt31axLFwEAPjNmmgAAACwgNAEAAFhAaAIAALCA0AQAAGABoQkAAMACQhMAAIAFhCYAAAALCE0AAAAWEJoAAAAsIDQBAABYQGgCAACwgNAEAABgAaEJAADAAkITAACABYQmAAAACwhNAAAAFhCaAAAALCA0AQAAWEBoAgAAsIDQBAAAYAGhCQAAwAJCEwAAgAWEJgAAAAsITQAAABYQmgAAACwgNAEAAFhAaAIAALCA0AQAAGABoQkAAMACQhMAAIAFhCYAAAALCE0AAAAWEJoAAAAsIDQBAABYEPTQ9N577+m+++5TQkKCoqOj9eUvf1kNDQ3muGEYWrFihVwul6KiojRlyhQdOnQoYBt+v19LlixRYmKiYmJiNGfOHB0/fjygxuv1yu12y+FwyOFwyO1268yZMwE1zc3Nmj17tmJiYpSYmKji4mJ1dXVdsWMHAABDR1BDk9fr1aRJkxQREaGdO3fqL3/5i5566il96UtfMmvWrFmjiooKVVZW6sCBA3I6nZo+fbrOnj1r1pSUlGjbtm2qqqpSXV2dzp07p/z8fPX09Jg1BQUFamxsVHV1taqrq9XY2Ci3222O9/T0aNasWero6FBdXZ2qqqq0detWlZaWfi7nAgAAhDabYRhGsHb++OOP6w9/+IN+//vfX3DcMAy5XC6VlJTosccek/TRrFJycrJWr16txYsXy+fzaeTIkXrxxRc1b948SdKJEyeUkpKiHTt2KC8vT4cPH1ZGRobq6+uVnZ0tSaqvr1dOTo7eeustpaena+fOncrPz1dLS4tcLpckqaqqSgsWLFBbW5vi4uIueTzt7e1yOBzy+XyW6hF6rn18e7BbuGzvrpoV7BYAYEiz+vc7qDNNr776qsaPH6+vf/3rSkpK0le+8hX98Ic/NMePHj0qj8ej3Nxcc53dbtfkyZO1Z88eSVJDQ4O6u7sDalwulzIzM82avXv3yuFwmIFJkiZMmCCHwxFQk5mZaQYmScrLy5Pf7w/4uPCT/H6/2tvbAxYAADA8BTU0vfPOO9qwYYPS0tL061//Wg899JCKi4v1wgsvSJI8Ho8kKTk5OeB9ycnJ5pjH41FkZKTi4+MvWpOUlNRv/0lJSQE1ffcTHx+vyMhIs6avlStXmvdIORwOpaSkXO4pAAAAQ0RQQ1Nvb69uvvlmlZeX6ytf+YoWL16swsJCbdiwIaDOZrMFvDYMo9+6vvrWXKh+IDWftHz5cvl8PnNpaWm5aE8AAGDoCmpoGjVqlDIyMgLWjR07Vs3NzZIkp9MpSf1metra2sxZIafTqa6uLnm93ovWnDx5st/+T506FVDTdz9er1fd3d39ZqA+ZrfbFRcXF7AAAIDhKaihadKkSTpy5EjAur/+9a8aPXq0JCk1NVVOp1O7du0yx7u6ulRbW6uJEydKkrKyshQRERFQ09raqqamJrMmJydHPp9P+/fvN2v27dsnn88XUNPU1KTW1lazpqamRna7XVlZWYN85AAAYKgJD+bOv/Wtb2nixIkqLy/X3LlztX//fj377LN69tlnJX30cVlJSYnKy8uVlpamtLQ0lZeXKzo6WgUFBZIkh8OhhQsXqrS0VAkJCRoxYoTKyso0btw4TZs2TdJHs1czZsxQYWGhNm7cKElatGiR8vPzlZ6eLknKzc1VRkaG3G631q5dq9OnT6usrEyFhYXMIAEAgOCGpq9+9avatm2bli9fru9973tKTU3V008/rW984xtmzbJly9TZ2amioiJ5vV5lZ2erpqZGsbGxZs369esVHh6uuXPnqrOzU1OnTtXmzZsVFhZm1mzZskXFxcXmt+zmzJmjyspKczwsLEzbt29XUVGRJk2apKioKBUUFGjdunWfw5kAAAChLqjPaRpueE7T0MdzmgDgn8+QeE4TAADAUEFoAgAAsIDQBAAAYAGhCQAAwAJCEwAAgAWEJgAAAAsITQAAABYQmgAAACwgNAEAAFhAaAIAALCA0AQAAGABoQkAAMACQhMAAIAFhCYAAAALCE0AAAAWEJoAAAAsIDQBAABYQGgCAACwgNAEAABgAaEJAADAAkITAACABYQmAAAACwhNAAAAFhCaAAAALCA0AQAAWEBoAgAAsIDQBAAAYAGhCQAAwAJCEwAAgAWEJgAAAAsITQAAABYQmgAAACwgNAEAAFhAaAIAALCA0AQAAGABoQkAAMACQhMAAIAFhCYAAAALCE0AAAAWEJoAAAAsIDQBAABYENTQtGLFCtlstoDF6XSa44ZhaMWKFXK5XIqKitKUKVN06NChgG34/X4tWbJEiYmJiomJ0Zw5c3T8+PGAGq/XK7fbLYfDIYfDIbfbrTNnzgTUNDc3a/bs2YqJiVFiYqKKi4vV1dV1xY4dAAAMLUGfabrhhhvU2tpqLm+++aY5tmbNGlVUVKiyslIHDhyQ0+nU9OnTdfbsWbOmpKRE27ZtU1VVlerq6nTu3Dnl5+erp6fHrCkoKFBjY6Oqq6tVXV2txsZGud1uc7ynp0ezZs1SR0eH6urqVFVVpa1bt6q0tPTzOQkAACDkhQe9gfDwgNmljxmGoaefflrf+c53dM8990iSnn/+eSUnJ+snP/mJFi9eLJ/Pp+eee04vvviipk2bJkn68Y9/rJSUFL322mvKy8vT4cOHVV1drfr6emVnZ0uSfvjDHyonJ0dHjhxRenq6ampq9Je//EUtLS1yuVySpKeeekoLFizQf//3fysuLu5zOhsAACBUBX2m6e2335bL5VJqaqruvfdevfPOO5Kko0ePyuPxKDc316y12+2aPHmy9uzZI0lqaGhQd3d3QI3L5VJmZqZZs3fvXjkcDjMwSdKECRPkcDgCajIzM83AJEl5eXny+/1qaGi4cgcPAACGjKDONGVnZ+uFF17QmDFjdPLkST355JOaOHGiDh06JI/HI0lKTk4OeE9ycrKOHTsmSfJ4PIqMjFR8fHy/mo/f7/F4lJSU1G/fSUlJATV99xMfH6/IyEiz5kL8fr/8fr/5ur293eqhAwCAISaooWnmzJnmf48bN045OTm67rrr9Pzzz2vChAmSJJvNFvAewzD6reurb82F6gdS09fKlSv13e9+96K9AACA4SHoH899UkxMjMaNG6e3337bvM+p70xPW1ubOSvkdDrV1dUlr9d70ZqTJ0/229epU6cCavrux+v1qru7u98M1CctX75cPp/PXFpaWi7ziAEAwFARUqHJ7/fr8OHDGjVqlFJTU+V0OrVr1y5zvKurS7W1tZo4caIkKSsrSxEREQE1ra2tampqMmtycnLk8/m0f/9+s2bfvn3y+XwBNU1NTWptbTVrampqZLfblZWV9an92u12xcXFBSwAAGB4CurHc2VlZZo9e7auueYatbW16cknn1R7e7vmz58vm82mkpISlZeXKy0tTWlpaSovL1d0dLQKCgokSQ6HQwsXLlRpaakSEhI0YsQIlZWVady4cea36caOHasZM2aosLBQGzdulCQtWrRI+fn5Sk9PlyTl5uYqIyNDbrdba9eu1enTp1VWVqbCwkKCEAAAkBTk0HT8+HH953/+p95//32NHDlSEyZMUH19vUaPHi1JWrZsmTo7O1VUVCSv16vs7GzV1NQoNjbW3Mb69esVHh6uuXPnqrOzU1OnTtXmzZsVFhZm1mzZskXFxcXmt+zmzJmjyspKczwsLEzbt29XUVGRJk2apKioKBUUFGjdunWf05kAAAChzmYYhhHsJoaL9vZ2ORwO+Xw+ZqiGqGsf3x7sFi7bu6tmBbsFABjSrP79Dql7mgAAAEIVoQkAAMACQhMAAIAFhCYAAAALCE0AAAAWEJoAAAAsIDQBAABYQGgCAACwgNAEAABgAaEJAADAAkITAACABYQmAAAACwhNAAAAFhCaAAAALCA0AQAAWEBoAgAAsIDQBAAAYAGhCQAAwAJCEwAAgAWEJgAAAAsITQAAABYQmgAAACwgNAEAAFhAaAIAALCA0AQAAGABoQkAAMACQhMAAIAFhCYAAAALCE0AAAAWEJoAAAAsIDQBAABYQGgCAACwgNAEAABgAaEJAADAAkITAACABYQmAAAACwhNAAAAFgwoNB09enSw+wAAAAhpAwpN119/vW677Tb9+Mc/1vnz5we7JwAAgJAzoND05z//WV/5yldUWloqp9OpxYsXa//+/YPdGwAAQMgYUGjKzMxURUWF3nvvPW3atEkej0e33HKLbrjhBlVUVOjUqVOD3ScAAEBQfaYbwcPDw3X33XfrZz/7mVavXq2///3vKisr09VXX637779fra2tlre1cuVK2Ww2lZSUmOsMw9CKFSvkcrkUFRWlKVOm6NChQwHv8/v9WrJkiRITExUTE6M5c+bo+PHjATVer1dut1sOh0MOh0Nut1tnzpwJqGlubtbs2bMVExOjxMREFRcXq6ur67LPCQAAGJ4+U2g6ePCgioqKNGrUKFVUVKisrEx///vf9frrr+u9997TnXfeaWk7Bw4c0LPPPqsbb7wxYP2aNWtUUVGhyspKHThwQE6nU9OnT9fZs2fNmpKSEm3btk1VVVWqq6vTuXPnlJ+fr56eHrOmoKBAjY2Nqq6uVnV1tRobG+V2u83xnp4ezZo1Sx0dHaqrq1NVVZW2bt2q0tLSz3J6AADAMGIzDMO43DdVVFRo06ZNOnLkiO644w49+OCDuuOOO3TVVf8/g/3tb3/Tv/3bv+nDDz+86LbOnTunm2++WT/4wQ/05JNP6stf/rKefvppGYYhl8ulkpISPfbYY5I+mlVKTk7W6tWrtXjxYvl8Po0cOVIvvvii5s2bJ0k6ceKEUlJStGPHDuXl5enw4cPKyMhQfX29srOzJUn19fXKycnRW2+9pfT0dO3cuVP5+flqaWmRy+WSJFVVVWnBggVqa2tTXFycpfPS3t4uh8Mhn89n+T0ILdc+vj3YLfzTeHfVrGC3AACSrP/9HtBM04YNG1RQUKDm5ma98sorys/PDwhMknTNNdfoueeeu+S2Hn74Yc2aNUvTpk0LWH/06FF5PB7l5uaa6+x2uyZPnqw9e/ZIkhoaGtTd3R1Q43K5lJmZadbs3btXDofDDEySNGHCBDkcjoCazMxMMzBJUl5envx+vxoaGqyeFgAAMIyFD+RNb7/99iVrIiMjNX/+/IvWVFVVqaGhQQcPHuw35vF4JEnJyckB65OTk3Xs2DGzJjIyUvHx8f1qPn6/x+NRUlJSv+0nJSUF1PTdT3x8vCIjI82aC/H7/fL7/ebr9vb2T60FAABD24BmmjZt2qSf//zn/db//Oc/1/PPP29pGy0tLXr00Ue1ZcsWfeELX/jUOpvNFvDaMIx+6/rqW3Oh+oHU9LVy5Urz5nKHw6GUlJSL9gUAAIauAYWmVatWKTExsd/6pKQklZeXW9pGQ0OD2tralJWVpfDwcIWHh6u2tlb/8z//o/DwcHPmp+9MT1tbmznmdDrV1dUlr9d70ZqTJ0/22/+pU6cCavrux+v1qru7u98M1CctX75cPp/PXFpaWiwdOwAAGHoGFJqOHTum1NTUfutHjx6t5uZmS9uYOnWq3nzzTTU2NprL+PHj9Y1vfEONjY3613/9VzmdTu3atct8T1dXl2prazVx4kRJUlZWliIiIgJqWltb1dTUZNbk5OTI5/MFPHxz37598vl8ATVNTU0Bj0ioqamR3W5XVlbWpx6D3W5XXFxcwAIAAIanAd3TlJSUpDfeeEPXXnttwPo///nPSkhIsLSN2NhYZWZmBqyLiYlRQkKCub6kpETl5eVKS0tTWlqaysvLFR0drYKCAkmSw+HQwoULVVpaqoSEBI0YMUJlZWUaN26ceWP52LFjNWPGDBUWFmrjxo2SpEWLFik/P1/p6emSpNzcXGVkZMjtdmvt2rU6ffq0ysrKVFhYSBACAACSBhia7r33XhUXFys2Nla33nqrJKm2tlaPPvqo7r333kFrbtmyZers7FRRUZG8Xq+ys7NVU1Oj2NhYs2b9+vUKDw/X3Llz1dnZqalTp2rz5s0KCwsza7Zs2aLi4mLzW3Zz5sxRZWWlOR4WFqbt27erqKhIkyZNUlRUlAoKCrRu3bpBOxYAADC0Deg5TV1dXXK73fr5z3+u8PCPcldvb6/uv/9+PfPMM4qMjBz0RocCntM09PGcps8Pz2kCECqs/v0e0ExTZGSkfvrTn+q//uu/9Oc//1lRUVEaN26cRo8ePeCGAQAAQtmAQtPHxowZozFjxgxWLwAAACFrQKGpp6dHmzdv1m9+8xu1tbWpt7c3YPz1118flOYAAABCxYBC06OPPqrNmzdr1qxZyszMvOTDJgEAAIa6AYWmqqoq/exnP9Mdd9wx2P0AAACEpAE93DIyMlLXX3/9YPcCAAAQsgYUmkpLS/X9739fA3haAQAAwJA0oI/n6urqtHv3bu3cuVM33HCDIiIiAsZffvnlQWkOAAAgVAwoNH3pS1/S3XffPdi9AAAAhKwBhaZNmzYNdh8AAAAhbUD3NEnShx9+qNdee00bN27U2bNnJUknTpzQuXPnBq05AACAUDGgmaZjx45pxowZam5ult/v1/Tp0xUbG6s1a9bo/PnzeuaZZwa7TwAAgKAa0EzTo48+qvHjx8vr9SoqKspcf/fdd+s3v/nNoDUHAAAQKgb87bk//OEPioyMDFg/evRovffee4PSGAAAQCgZ0ExTb2+venp6+q0/fvy4YmNjP3NTAAAAoWZAoWn69Ol6+umnzdc2m03nzp3TE088wU+rAACAYWlAH8+tX79et912mzIyMnT+/HkVFBTo7bffVmJiol566aXB7hEAACDoBhSaXC6XGhsb9dJLL+mPf/yjent7tXDhQn3jG98IuDEcAABguBhQaJKkqKgoPfDAA3rggQcGsx8AAICQNKDQ9MILL1x0/P777x9QMwAAAKFqQKHp0UcfDXjd3d2tf/zjH4qMjFR0dDShCQAADDsD+vac1+sNWM6dO6cjR47olltu4UZwAAAwLA34t+f6SktL06pVq/rNQgEAAAwHgxaaJCksLEwnTpwYzE0CAACEhAHd0/Tqq68GvDYMQ62traqsrNSkSZMGpTEAAIBQMqDQdNdddwW8ttlsGjlypG6//XY99dRTg9EXAABASBlQaOrt7R3sPgAAAELaoN7TBAAAMFwNaKZp6dKllmsrKioGsgsAAICQMqDQ9Kc//Ul//OMf9eGHHyo9PV2S9Ne//lVhYWG6+eabzTqbzTY4XQIAAATZgELT7NmzFRsbq+eff17x8fGSPnrg5Te/+U197WtfU2lp6aA2CQAAEGw2wzCMy33Tv/zLv6impkY33HBDwPqmpibl5ub+0z6rqb29XQ6HQz6fT3FxccFuBwNw7ePbg90CQti7q2YFuwUAV4DVv98DuhG8vb1dJ0+e7Le+ra1NZ8+eHcgmAQAAQtqAQtPdd9+tb37zm/rFL36h48eP6/jx4/rFL36hhQsX6p577hnsHgEAAIJuQPc0PfPMMyorK9N9992n7u7ujzYUHq6FCxdq7dq1g9ogAABAKBhQaIqOjtYPfvADrV27Vn//+99lGIauv/56xcTEDHZ/AAAAIeEzPdyytbVVra2tGjNmjGJiYjSAe8oBAACGhAGFpg8++EBTp07VmDFjdMcdd6i1tVWS9OCDD/K4AQAAMCwNKDR961vfUkREhJqbmxUdHW2unzdvnqqrqwetOQAAgFAxoHuaampq9Otf/1pXX311wPq0tDQdO3ZsUBoDAAAIJQOaaero6AiYYfrY+++/L7vd/pmbAgAACDUDCk233nqrXnjhBfO1zWZTb2+v1q5dq9tuu23QmgMAAAgVAwpNa9eu1caNGzVz5kx1dXVp2bJlyszM1O9+9zutXr3a8nY2bNigG2+8UXFxcYqLi1NOTo527txpjhuGoRUrVsjlcikqKkpTpkzRoUOHArbh9/u1ZMkSJSYmKiYmRnPmzNHx48cDarxer9xutxwOhxwOh9xut86cORNQ09zcrNmzZysmJkaJiYkqLi5WV1fX5Z8cAAAwLA0oNGVkZOiNN97Qv//7v2v69Onq6OjQPffcoz/96U+67rrrLG/n6quv1qpVq3Tw4EEdPHhQt99+u+68804zGK1Zs0YVFRWqrKzUgQMH5HQ6NX369ICfaikpKdG2bdtUVVWluro6nTt3Tvn5+erp6TFrCgoK1NjYqOrqalVXV6uxsVFut9sc7+np0axZs9TR0aG6ujpVVVVp69atfBMQAACYLvsHe7u7u5Wbm6uNGzdqzJgxg97QiBEjtHbtWj3wwANyuVwqKSnRY489JumjWaXk5GStXr1aixcvls/n08iRI/Xiiy9q3rx5kqQTJ04oJSVFO3bsUF5eng4fPqyMjAzV19crOztbklRfX6+cnBy99dZbSk9P186dO5Wfn6+Wlha5XC5JUlVVlRYsWKC2tjbLP77LD/YOffxgLy6GH+wFhqcr9oO9ERERampqks1m+0wN9tXT06Oqqip1dHQoJydHR48elcfjUW5urlljt9s1efJk7dmzR5LU0NBghriPuVwuZWZmmjV79+6Vw+EwA5MkTZgwQQ6HI6AmMzPTDEySlJeXJ7/fr4aGhk/t2e/3q729PWABAADD04A+nrv//vv13HPPDUoDb775pr74xS/KbrfroYce0rZt25SRkSGPxyNJSk5ODqhPTk42xzwejyIjIxUfH3/RmqSkpH77TUpKCqjpu5/4+HhFRkaaNReycuVK8z4ph8OhlJSUyzx6AAAwVAzoOU1dXV360Y9+pF27dmn8+PH9fnOuoqLC8rbS09PV2NioM2fOaOvWrZo/f75qa2vN8b4zWoZhXHKWq2/NheoHUtPX8uXLtXTpUvN1e3s7wQkAgGHqskLTO++8o2uvvVZNTU26+eabJUl//etfA2ou92O7yMhIXX/99ZKk8ePH68CBA/r+979v3sfk8Xg0atQos76trc2cFXI6nerq6pLX6w2YbWpra9PEiRPNmpMnT/bb76lTpwK2s2/fvoBxr9er7u7ufjNQn2S323kuFQAA/yQu6+O5tLQ0vf/++9q9e7d2796tpKQkVVVVma93796t119//TM1ZBiG/H6/UlNT5XQ6tWvXLnOsq6tLtbW1ZiDKyspSREREQE1ra6uamprMmpycHPl8Pu3fv9+s2bdvn3w+X0BNU1OT+Rt60kdPPbfb7crKyvpMxwMAAIaHy5pp6vtFu507d6qjo2PAO//2t7+tmTNnKiUlRWfPnlVVVZV++9vfqrq6WjabTSUlJSovL1daWprS0tJUXl6u6OhoFRQUSJIcDocWLlyo0tJSJSQkaMSIESorK9O4ceM0bdo0SdLYsWM1Y8YMFRYWauPGjZKkRYsWKT8/X+np6ZKk3NxcZWRkyO12a+3atTp9+rTKyspUWFjIt+AAAICkAd7T9LHLfFpBPydPnpTb7VZra6scDoduvPFGVVdXa/r06ZKkZcuWqbOzU0VFRfJ6vcrOzlZNTY1iY2PNbaxfv17h4eGaO3euOjs7NXXqVG3evFlhYWFmzZYtW1RcXGx+y27OnDmqrKw0x8PCwrR9+3YVFRVp0qRJioqKUkFBgdatW/eZjg8AAAwfl/WcprCwMHk8Ho0cOVKSFBsbqzfeeEOpqalXrMGhhOc0DX08pwkXw3OagOHJ6t/vy/54bsGCBebNz+fPn9dDDz3U79tzL7/88gBaBgAACF2XFZrmz58f8Pq+++4b1GYAAABC1WWFpk2bNl2pPgAAAELagJ4IDgAA8M+G0AQAAGABoQkAAMACQhMAAIAFhCYAAAALCE0AAAAWEJoAAAAsIDQBAABYQGgCAACwgNAEAABgAaEJAADAAkITAACABYQmAAAACwhNAAAAFhCaAAAALCA0AQAAWEBoAgAAsIDQBAAAYAGhCQAAwAJCEwAAgAWEJgAAAAsITQAAABYQmgAAACwgNAEAAFhAaAIAALCA0AQAAGBBeLAbwPB17ePbg90CAACDhpkmAAAACwhNAAAAFhCaAAAALCA0AQAAWEBoAgAAsIDQBAAAYAGhCQAAwAJCEwAAgAWEJgAAAAsITQAAABYQmgAAACwIamhauXKlvvrVryo2NlZJSUm66667dOTIkYAawzC0YsUKuVwuRUVFacqUKTp06FBAjd/v15IlS5SYmKiYmBjNmTNHx48fD6jxer1yu91yOBxyOBxyu906c+ZMQE1zc7Nmz56tmJgYJSYmqri4WF1dXVfk2AEAwNAS1NBUW1urhx9+WPX19dq1a5c+/PBD5ebmqqOjw6xZs2aNKioqVFlZqQMHDsjpdGr69Ok6e/asWVNSUqJt27apqqpKdXV1OnfunPLz89XT02PWFBQUqLGxUdXV1aqurlZjY6Pcbrc53tPTo1mzZqmjo0N1dXWqqqrS1q1bVVpa+vmcDAAAENJshmEYwW7iY6dOnVJSUpJqa2t16623yjAMuVwulZSU6LHHHpP00axScnKyVq9ercWLF8vn82nkyJF68cUXNW/ePEnSiRMnlJKSoh07digvL0+HDx9WRkaG6uvrlZ2dLUmqr69XTk6O3nrrLaWnp2vnzp3Kz89XS0uLXC6XJKmqqkoLFixQW1ub4uLiLtl/e3u7HA6HfD6fpfrh7trHtwe7BWBQvbtqVrBbAHAFWP37HVL3NPl8PknSiBEjJElHjx6Vx+NRbm6uWWO32zV58mTt2bNHktTQ0KDu7u6AGpfLpczMTLNm7969cjgcZmCSpAkTJsjhcATUZGZmmoFJkvLy8uT3+9XQ0HDBfv1+v9rb2wMWAAAwPIVMaDIMQ0uXLtUtt9yizMxMSZLH45EkJScnB9QmJyebYx6PR5GRkYqPj79oTVJSUr99JiUlBdT03U98fLwiIyPNmr5Wrlxp3iPlcDiUkpJyuYcNAACGiJAJTY888ojeeOMNvfTSS/3GbDZbwGvDMPqt66tvzYXqB1LzScuXL5fP5zOXlpaWi/YEAACGrpAITUuWLNGrr76q3bt36+qrrzbXO51OSeo309PW1mbOCjmdTnV1dcnr9V605uTJk/32e+rUqYCavvvxer3q7u7uNwP1Mbvdrri4uIAFAAAMT0ENTYZh6JFHHtHLL7+s119/XampqQHjqampcjqd2rVrl7muq6tLtbW1mjhxoiQpKytLERERATWtra1qamoya3JycuTz+bR//36zZt++ffL5fAE1TU1Nam1tNWtqampkt9uVlZU1+AcPAACGlPBg7vzhhx/WT37yE/3yl79UbGysOdPjcDgUFRUlm82mkpISlZeXKy0tTWlpaSovL1d0dLQKCgrM2oULF6q0tFQJCQkaMWKEysrKNG7cOE2bNk2SNHbsWM2YMUOFhYXauHGjJGnRokXKz89Xenq6JCk3N1cZGRlyu91au3atTp8+rbKyMhUWFjKDBAAAghuaNmzYIEmaMmVKwPpNmzZpwYIFkqRly5aps7NTRUVF8nq9ys7OVk1NjWJjY8369evXKzw8XHPnzlVnZ6emTp2qzZs3KywszKzZsmWLiouLzW/ZzZkzR5WVleZ4WFiYtm/frqKiIk2aNElRUVEqKCjQunXrrtDRAwCAoSSkntM01PGcpkA8pwnDDc9pAoanIfmcJgAAgFBFaAIAALCA0AQAAGABoQkAAMACQhMAAIAFhCYAAAALCE0AAAAWEJoAAAAsIDQBAABYQGgCAACwgNAEAABgAaEJAADAAkITAACABYQmAAAACwhNAAAAFhCaAAAALCA0AQAAWEBoAgAAsIDQBAAAYAGhCQAAwAJCEwAAgAWEJgAAAAsITQAAABYQmgAAACwgNAEAAFhAaAIAALCA0AQAAGABoQkAAMACQhMAAIAFhCYAAAALCE0AAAAWEJoAAAAsIDQBAABYQGgCAACwgNAEAABgAaEJAADAAkITAACABYQmAAAACwhNAAAAFhCaAAAALAgPdgMAMFRc+/j2YLdw2d5dNSvYLQDDRlBnmn73u99p9uzZcrlcstlseuWVVwLGDcPQihUr5HK5FBUVpSlTpujQoUMBNX6/X0uWLFFiYqJiYmI0Z84cHT9+PKDG6/XK7XbL4XDI4XDI7XbrzJkzATXNzc2aPXu2YmJilJiYqOLiYnV1dV2JwwYAAENQUENTR0eHbrrpJlVWVl5wfM2aNaqoqFBlZaUOHDggp9Op6dOn6+zZs2ZNSUmJtm3bpqqqKtXV1encuXPKz89XT0+PWVNQUKDGxkZVV1erurpajY2Ncrvd5nhPT49mzZqljo4O1dXVqaqqSlu3blVpaemVO3gAADCk2AzDMILdhCTZbDZt27ZNd911l6SPZplcLpdKSkr02GOPSfpoVik5OVmrV6/W4sWL5fP5NHLkSL344ouaN2+eJOnEiRNKSUnRjh07lJeXp8OHDysjI0P19fXKzs6WJNXX1ysnJ0dvvfWW0tPTtXPnTuXn56ulpUUul0uSVFVVpQULFqitrU1xcXGWjqG9vV0Oh0M+n8/ye4azofhRBjDc8PEccGlW/36H7I3gR48elcfjUW5urrnObrdr8uTJ2rNnjySpoaFB3d3dATUul0uZmZlmzd69e+VwOMzAJEkTJkyQw+EIqMnMzDQDkyTl5eXJ7/eroaHhU3v0+/1qb28PWAAAwPAUsqHJ4/FIkpKTkwPWJycnm2Mej0eRkZGKj4+/aE1SUlK/7SclJQXU9N1PfHy8IiMjzZoLWblypXmflMPhUEpKymUeJQAAGCpCNjR9zGazBbw2DKPfur761lyofiA1fS1fvlw+n89cWlpaLtoXAAAYukI2NDmdTknqN9PT1tZmzgo5nU51dXXJ6/VetObkyZP9tn/q1KmAmr778Xq96u7u7jcD9Ul2u11xcXEBCwAAGJ5CNjSlpqbK6XRq165d5rquri7V1tZq4sSJkqSsrCxFREQE1LS2tqqpqcmsycnJkc/n0/79+82affv2yefzBdQ0NTWptbXVrKmpqZHdbldWVtYVPU4AADA0BPXhlufOndPf/vY38/XRo0fV2NioESNG6JprrlFJSYnKy8uVlpamtLQ0lZeXKzo6WgUFBZIkh8OhhQsXqrS0VAkJCRoxYoTKyso0btw4TZs2TZI0duxYzZgxQ4WFhdq4caMkadGiRcrPz1d6erokKTc3VxkZGXK73Vq7dq1Onz6tsrIyFRYWMnsEAAAkBTk0HTx4ULfddpv5eunSpZKk+fPna/PmzVq2bJk6OztVVFQkr9er7Oxs1dTUKDY21nzP+vXrFR4errlz56qzs1NTp07V5s2bFRYWZtZs2bJFxcXF5rfs5syZE/BsqLCwMG3fvl1FRUWaNGmSoqKiVFBQoHXr1l3pUwAAAIaIkHlO03DAc5oC8ZwmIPh4ThNwaUP+OU0AAAChhNAEAABgAaEJAADAAkITAACABYQmAAAACwhNAAAAFhCaAAAALCA0AQAAWEBoAgAAsIDQBAAAYAGhCQAAwAJCEwAAgAWEJgAAAAsITQAAABYQmgAAACwgNAEAAFhAaAIAALCA0AQAAGABoQkAAMACQhMAAIAFhCYAAAALCE0AAAAWEJoAAAAsIDQBAABYQGgCAACwgNAEAABgAaEJAADAAkITAACABYQmAAAACwhNAAAAFhCaAAAALCA0AQAAWEBoAgAAsIDQBAAAYAGhCQAAwILwYDcAALhyrn18e7BbuGzvrpoV7BaAC2KmCQAAwAJCEwAAgAV8PDdEDMUpdgAAhhNCEwAgpAzFfyRyH9Y/Bz6eAwAAsIDQ1McPfvADpaam6gtf+IKysrL0+9//PtgtAQCAEEBo+oSf/vSnKikp0Xe+8x396U9/0te+9jXNnDlTzc3NwW4NAAAEGaHpEyoqKrRw4UI9+OCDGjt2rJ5++mmlpKRow4YNwW4NAAAEGTeC/5+uri41NDTo8ccfD1ifm5urPXv2XPA9fr9ffr/ffO3z+SRJ7e3tg95fr/8fg75NAMDguOZbPw92C5et6bt5wW4hZHz8d9swjIvWEZr+z/vvv6+enh4lJycHrE9OTpbH47nge1auXKnvfve7/danpKRckR4BABgsjqeD3UHoOXv2rBwOx6eOE5r6sNlsAa8Nw+i37mPLly/X0qVLzde9vb06ffq0EhISPvU9Q1l7e7tSUlLU0tKiuLi4YLczpHEuBxfnc/BwLgcX53PwXMlzaRiGzp49K5fLddE6QtP/SUxMVFhYWL9Zpba2tn6zTx+z2+2y2+0B6770pS9dqRZDRlxcHP/zDxLO5eDifA4ezuXg4nwOnit1Li82w/QxbgT/P5GRkcrKytKuXbsC1u/atUsTJ04MUlcAACBUMNP0CUuXLpXb7db48eOVk5OjZ599Vs3NzXrooYeC3RoAAAgyQtMnzJs3Tx988IG+973vqbW1VZmZmdqxY4dGjx4d7NZCgt1u1xNPPNHvI0lcPs7l4OJ8Dh7O5eDifA6eUDiXNuNS368DAAAA9zQBAABYQWgCAACwgNAEAABgAaEJAADAAkITLmrFihWy2WwBi9PpDHZbQ8bvfvc7zZ49Wy6XSzabTa+88krAuGEYWrFihVwul6KiojRlyhQdOnQoOM2GuEudywULFvS7VidMmBCcZkPcypUr9dWvflWxsbFKSkrSXXfdpSNHjgTUcG1aZ+V8cn1as2HDBt14443mAyxzcnK0c+dOczzY1yWhCZd0ww03qLW11VzefPPNYLc0ZHR0dOimm25SZWXlBcfXrFmjiooKVVZW6sCBA3I6nZo+fbrOnj37OXca+i51LiVpxowZAdfqjh07PscOh47a2lo9/PDDqq+v165du/Thhx8qNzdXHR0dZg3XpnVWzqfE9WnF1VdfrVWrVungwYM6ePCgbr/9dt15551mMAr6dWkAF/HEE08YN910U7DbGBYkGdu2bTNf9/b2Gk6n01i1apW57vz584bD4TCeeeaZIHQ4dPQ9l4ZhGPPnzzfuvPPOoPQz1LW1tRmSjNraWsMwuDY/q77n0zC4Pj+L+Ph440c/+lFIXJfMNOGS3n77bblcLqWmpuree+/VO++8E+yWhoWjR4/K4/EoNzfXXGe32zV58mTt2bMniJ0NXb/97W+VlJSkMWPGqLCwUG1tbcFuaUjw+XySpBEjRkji2vys+p7Pj3F9Xp6enh5VVVWpo6NDOTk5IXFdEppwUdnZ2XrhhRf061//Wj/84Q/l8Xg0ceJEffDBB8Fubcj7+Meh+/4gdHJycr8fjsalzZw5U1u2bNHrr7+up556SgcOHNDtt98uv98f7NZCmmEYWrp0qW655RZlZmZK4tr8LC50PiWuz8vx5ptv6otf/KLsdrseeughbdu2TRkZGSFxXfIzKriomTNnmv89btw45eTk6LrrrtPzzz+vpUuXBrGz4cNmswW8Ngyj3zpc2rx588z/zszM1Pjx4zV69Ght375d99xzTxA7C22PPPKI3njjDdXV1fUb49q8fJ92Prk+rUtPT1djY6POnDmjrVu3av78+aqtrTXHg3ldMtOEyxITE6Nx48bp7bffDnYrQ97H30Ls+y+ktra2fv+SwuUbNWqURo8ezbV6EUuWLNGrr76q3bt36+qrrzbXc20OzKedzwvh+vx0kZGRuv766zV+/HitXLlSN910k77//e+HxHVJaMJl8fv9Onz4sEaNGhXsVoa81NRUOZ1O7dq1y1zX1dWl2tpaTZw4MYidDQ8ffPCBWlpauFYvwDAMPfLII3r55Zf1+uuvKzU1NWCca/PyXOp8XgjXp3WGYcjv94fEdcnHc7iosrIyzZ49W9dcc43a2tr05JNPqr29XfPnzw92a0PCuXPn9Le//c18ffToUTU2NmrEiBG65pprVFJSovLycqWlpSktLU3l5eWKjo5WQUFBELsOTRc7lyNGjNCKFSv0H//xHxo1apTeffddffvb31ZiYqLuvvvuIHYdmh5++GH95Cc/0S9/+UvFxsaa/3J3OByKioqSzWbj2rwMlzqf586d4/q06Nvf/rZmzpyplJQUnT17VlVVVfrtb3+r6urq0LguP5fv6GHImjdvnjFq1CgjIiLCcLlcxj333GMcOnQo2G0NGbt37zYk9Vvmz59vGMZHX+1+4oknDKfTadjtduPWW2813nzzzeA2HaIudi7/8Y9/GLm5ucbIkSONiIgI45prrjHmz59vNDc3B7vtkHSh8yjJ2LRpk1nDtWndpc4n16d1DzzwgDF69GgjMjLSGDlypDF16lSjpqbGHA/2dWkzDMP4fOIZAADA0MU9TQAAABYQmgAAACwgNAEAAFhAaAIAALCA0AQAAGABoQkAAMACQhMAAIAFhCYAAAALCE0AAAAWEJoAAAAsIDQBAABYQGgCAACw4P8BLhkOTTKzHhYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_df.total_lines.plot.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "09462932",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(180040, 30212, 30135)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sentences = train_df['text'].tolist()\n",
    "val_sentences = val_df['text'].tolist()\n",
    "test_sentences = test_df['text'].tolist()\n",
    "\n",
    "len(train_sentences), len(val_sentences), len(test_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4750b521",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['to investigate the efficacy of @ weeks of daily low-dose oral prednisolone in improving pain , mobility , and systemic low-grade inflammation in the short term and whether the effect would be sustained at @ weeks in older adults with moderate to severe knee osteoarthritis ( oa ) .',\n",
       " 'a total of @ patients with primary knee oa were randomized @:@ ; @ received @ mg/day of prednisolone and @ received placebo for @ weeks .',\n",
       " 'outcome measures included pain reduction and improvement in function scores and systemic inflammation markers .',\n",
       " 'pain was assessed using the visual analog pain scale ( @-@ mm ) .',\n",
       " 'secondary outcome measures included the western ontario and mcmaster universities osteoarthritis index scores , patient global assessment ( pga ) of the severity of knee oa , and @-min walk distance ( @mwd ) .',\n",
       " 'serum levels of interleukin @ ( il-@ ) , il-@ , tumor necrosis factor ( tnf ) - , and high-sensitivity c-reactive protein ( hscrp ) were measured .',\n",
       " 'there was a clinically relevant reduction in the intervention group compared to the placebo group for knee pain , physical function , pga , and @mwd at @ weeks .',\n",
       " 'the mean difference between treatment arms ( @ % ci ) was @ ( @-@ @ ) , p < @ ; @ ( @-@ @ ) , p < @ ; @ ( @-@ @ ) , p < @ ; and @ ( @-@ @ ) , p < @ , respectively .',\n",
       " 'further , there was a clinically relevant reduction in the serum levels of il-@ , il-@ , tnf - , and hscrp at @ weeks in the intervention group when compared to the placebo group .',\n",
       " 'these differences remained significant at @ weeks .']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sentences[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "da679826",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "one_hot_encoder = OneHotEncoder(sparse=False)\n",
    "train_labels_one_hot = one_hot_encoder.fit_transform(train_df['target'].to_numpy().reshape(-1,1))\n",
    "val_labels_one_hot = one_hot_encoder.fit_transform(val_df['target'].to_numpy().reshape(-1,1))\n",
    "test_labels_one_hot = one_hot_encoder.fit_transform(test_df['target'].to_numpy().reshape(-1,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "39ce2348",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(180040, 5), dtype=float64, numpy=\n",
       "array([[0., 0., 0., 1., 0.],\n",
       "       [0., 0., 1., 0., 0.],\n",
       "       [0., 0., 1., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., 0., 1.],\n",
       "       [0., 1., 0., 0., 0.],\n",
       "       [0., 1., 0., 0., 0.]])>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(train_labels_one_hot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9f3cb72b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3, 2, 2, ..., 4, 1, 1])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "label_encoder = LabelEncoder()\n",
    "train_labels_encoded = label_encoder.fit_transform(train_df['target'].to_numpy())\n",
    "val_labels_encoded = label_encoder.transform(val_df['target'].to_numpy())\n",
    "test_labels_encoded = label_encoder.transform(test_df['target'].to_numpy())\n",
    "\n",
    "train_labels_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6642f48e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5,\n",
       " array(['BACKGROUND', 'CONCLUSIONS', 'METHODS', 'OBJECTIVE', 'RESULTS'],\n",
       "       dtype=object))"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_classes = len(label_encoder.classes_)\n",
    "class_names = label_encoder.classes_\n",
    "num_classes, class_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f40584e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#----Models----\n",
    "#1 - Naive Bayes with TF-IDF encoder\n",
    "#2 - Conv1D with token embeddings\n",
    "#3 - TensorFlow Hub Pretrained Feature Extractor\n",
    "#4 - Conv1D with character embeddings\n",
    "#5 - Pretrained token embeddings (same as 2) + character embeddings (same as 3)\n",
    "#6 - Pretrained token embeddings + character embeddings + positional embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8c4f1339",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;tf-idf&#x27;, TfidfVectorizer()), (&#x27;clf&#x27;, MultinomialNB())])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;tf-idf&#x27;, TfidfVectorizer()), (&#x27;clf&#x27;, MultinomialNB())])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfVectorizer</label><div class=\"sk-toggleable__content\"><pre>TfidfVectorizer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultinomialNB</label><div class=\"sk-toggleable__content\"><pre>MultinomialNB()</pre></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "Pipeline(steps=[('tf-idf', TfidfVectorizer()), ('clf', MultinomialNB())])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "model_0 = Pipeline([\n",
    "    ('tf-idf', TfidfVectorizer()),\n",
    "    ('clf', MultinomialNB())\n",
    "])\n",
    "\n",
    "model_0.fit(\n",
    "    train_sentences,\n",
    "    train_labels_encoded\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "15cc0222",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7218323844829869"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_0.score(val_sentences, val_labels_encoded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6788ab35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4, 1, 3, ..., 4, 4, 1])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseline_preds = model_0.predict(val_sentences)\n",
    "baseline_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f591a4a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
    "\n",
    "def calculate_results(y_true, y_pred):\n",
    "    model_accuracy = accuracy_score(y_true, y_pred)*100\n",
    "    model_precision, model_recall, model_f1, _ = precision_recall_fscore_support(y_true, y_pred, average='weighted')\n",
    "    model_results = {\"accuracy\": model_accuracy,\n",
    "                     \"precision\": model_precision,\n",
    "                     \"recall\": model_recall,\n",
    "                     \"f1-score\": model_f1}\n",
    "    return model_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "eed26e96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 72.1832384482987,\n",
       " 'precision': 0.7186466952323352,\n",
       " 'recall': 0.7218323844829869,\n",
       " 'f1-score': 0.6989250353450294}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseline_results = calculate_results(val_labels_encoded, baseline_preds)\n",
    "baseline_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "ead09401",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26.0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Model-1\n",
    "#Finding average length of sentences in the training data\n",
    "\n",
    "sent_lens = [len(sentence.split()) for sentence in train_sentences]\n",
    "avg_sent_len = np.round(np.mean(sent_lens))\n",
    "avg_sent_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c3dd1f49",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([7.3163e+04, 8.1733e+04, 1.8772e+04, 4.5680e+03, 1.2400e+03,\n",
       "        3.5400e+02, 1.1600e+02, 4.5000e+01, 1.7000e+01, 1.2000e+01,\n",
       "        1.2000e+01, 3.0000e+00, 3.0000e+00, 1.0000e+00, 1.0000e+00]),\n",
       " array([  1.        ,  20.66666667,  40.33333333,  60.        ,\n",
       "         79.66666667,  99.33333333, 119.        , 138.66666667,\n",
       "        158.33333333, 178.        , 197.66666667, 217.33333333,\n",
       "        237.        , 256.66666667, 276.33333333, 296.        ]),\n",
       " <BarContainer object of 15 artists>)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjoAAAGdCAYAAAAbudkLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA0iUlEQVR4nO3df1BV94H//xcFuUUKpyjC9W5IQrcslWKyKeki2lY3KuiKtJNOtaV7RycWTUlk7waqsdndms4W/BVNt7SpsZmYGrO38xlLN1OVQjYJLauooWErxqTZiQlYuWLj9YKE3kvwfP/IN6e9YIxXTdC3z8fMmZFzXuee93nPmeE1b++9xNm2bQsAAMBAHxnrAQAAAHxQKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGMljPUAxtK5c+d04sQJpaSkKC4ubqyHAwAALoJt2+rv75fH49FHPnLhNZvruuicOHFCWVlZYz0MAABwCbq7u3XDDTdcMHNdF52UlBRJ70xUamrqGI8GAABcjL6+PmVlZTm/xy/kui467/53VWpqKkUHAIBrzMW87YQ3IwMAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYK2GsB4Ar7+b7d4/JdV9ft2BMrgsAwHthRQcAABiLogMAAIxF0QEAAMaKqei8/fbb+pd/+RdlZ2crKSlJn/jEJ/Td735X586dczK2bWvt2rXyeDxKSkrSrFmzdOTIkajXCYfDWrlypdLT05WcnKyysjIdP348KhMMBuX1emVZlizLktfr1ZkzZ6IyXV1dWrhwoZKTk5Wenq6qqipFIpEYpwAAAJgqpqKzfv16/fjHP1Z9fb2OHj2qDRs2aOPGjfrBD37gZDZs2KDNmzervr5ehw4dktvt1ty5c9Xf3+9kfD6fGhoa5Pf71draqrNnz6q0tFTDw8NOpry8XB0dHWpsbFRjY6M6Ojrk9Xqd48PDw1qwYIEGBgbU2toqv9+vXbt2qbq6+nLmAwAAGCTOtm37YsOlpaXKzMzUY4895uz78pe/rPHjx2vHjh2ybVsej0c+n0+rV6+W9M7qTWZmptavX68VK1YoFApp0qRJ2rFjhxYvXixJOnHihLKysrRnzx6VlJTo6NGjysvLU1tbmwoLCyVJbW1tKioq0ssvv6zc3Fzt3btXpaWl6u7ulsfjkST5/X4tXbpUvb29Sk1Nfd/76evrk2VZCoVCF5W/VvCpKwCAyWL5/R3Tis7nPvc5/fd//7d+//vfS5L+93//V62trfqHf/gHSdKxY8cUCARUXFzsnONyuTRz5kzt27dPktTe3q6hoaGojMfjUX5+vpPZv3+/LMtySo4kTZs2TZZlRWXy8/OdkiNJJSUlCofDam9vP+/4w+Gw+vr6ojYAAGCumL5HZ/Xq1QqFQvrUpz6l+Ph4DQ8P63vf+56+9rWvSZICgYAkKTMzM+q8zMxMvfHGG04mMTFRaWlpozLvnh8IBJSRkTHq+hkZGVGZkddJS0tTYmKikxmprq5ODz74YCy3DAAArmExrej87Gc/05NPPqmnnnpKv/3tb/XEE09o06ZNeuKJJ6JycXFxUT/btj1q30gjM+fLX0rmL61Zs0ahUMjZuru7LzgmAABwbYtpRedb3/qW7r//fn31q1+VJE2dOlVvvPGG6urqtGTJErndbknvrLZMnjzZOa+3t9dZfXG73YpEIgoGg1GrOr29vZo+fbqTOXny5Kjrnzp1Kup1Dhw4EHU8GAxqaGho1ErPu1wul1wuVyy3DAAArmExrei89dZb+shHok+Jj493Pl6enZ0tt9ut5uZm53gkElFLS4tTYgoKCjRu3LioTE9Pjzo7O51MUVGRQqGQDh486GQOHDigUCgUlens7FRPT4+TaWpqksvlUkFBQSy3BQAADBXTis7ChQv1ve99TzfeeKM+/elP68UXX9TmzZt11113SXrnv5J8Pp9qa2uVk5OjnJwc1dbWavz48SovL5ckWZalZcuWqbq6WhMnTtSECRNUU1OjqVOnas6cOZKkKVOmaN68eaqoqNDWrVslScuXL1dpaalyc3MlScXFxcrLy5PX69XGjRt1+vRp1dTUqKKiwqhPUAEAgEsXU9H5wQ9+oH/9139VZWWlent75fF4tGLFCv3bv/2bk1m1apUGBwdVWVmpYDCowsJCNTU1KSUlxcls2bJFCQkJWrRokQYHBzV79mxt375d8fHxTmbnzp2qqqpyPp1VVlam+vp653h8fLx2796tyspKzZgxQ0lJSSovL9emTZsueTIAAIBZYvoeHdPwPTpXFt+jAwD4MHxg36MDAABwLaHoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMFbCWA/AZDffv3ushwAAwHWNFR0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYKyYis7NN9+suLi4Uds999wjSbJtW2vXrpXH41FSUpJmzZqlI0eORL1GOBzWypUrlZ6eruTkZJWVlen48eNRmWAwKK/XK8uyZFmWvF6vzpw5E5Xp6urSwoULlZycrPT0dFVVVSkSiVzCFAAAAFPFVHQOHTqknp4eZ2tubpYkfeUrX5EkbdiwQZs3b1Z9fb0OHTokt9utuXPnqr+/33kNn8+nhoYG+f1+tba26uzZsyotLdXw8LCTKS8vV0dHhxobG9XY2KiOjg55vV7n+PDwsBYsWKCBgQG1trbK7/dr165dqq6uvqzJAAAAZomzbdu+1JN9Pp9++ctf6tVXX5UkeTwe+Xw+rV69WtI7qzeZmZlav369VqxYoVAopEmTJmnHjh1avHixJOnEiRPKysrSnj17VFJSoqNHjyovL09tbW0qLCyUJLW1tamoqEgvv/yycnNztXfvXpWWlqq7u1sej0eS5Pf7tXTpUvX29io1NfWixt/X1yfLshQKhS76nFhcb3/r6vV1C8Z6CACA60Asv78v+T06kUhETz75pO666y7FxcXp2LFjCgQCKi4udjIul0szZ87Uvn37JEnt7e0aGhqKyng8HuXn5zuZ/fv3y7Isp+RI0rRp02RZVlQmPz/fKTmSVFJSonA4rPb29vccczgcVl9fX9QGAADMdclF5xe/+IXOnDmjpUuXSpICgYAkKTMzMyqXmZnpHAsEAkpMTFRaWtoFMxkZGaOul5GREZUZeZ20tDQlJiY6mfOpq6tz3vdjWZaysrJiuGMAAHCtueSi89hjj2n+/PlRqyqSFBcXF/Wzbduj9o00MnO+/KVkRlqzZo1CoZCzdXd3X3BcAADg2nZJReeNN97QM888o2984xvOPrfbLUmjVlR6e3ud1Re3261IJKJgMHjBzMmTJ0dd89SpU1GZkdcJBoMaGhoatdLzl1wul1JTU6M2AABgrksqOo8//rgyMjK0YMGf33yanZ0tt9vtfBJLeud9PC0tLZo+fbokqaCgQOPGjYvK9PT0qLOz08kUFRUpFArp4MGDTubAgQMKhUJRmc7OTvX09DiZpqYmuVwuFRQUXMotAQAAAyXEesK5c+f0+OOPa8mSJUpI+PPpcXFx8vl8qq2tVU5OjnJyclRbW6vx48ervLxckmRZlpYtW6bq6mpNnDhREyZMUE1NjaZOnao5c+ZIkqZMmaJ58+apoqJCW7dulSQtX75cpaWlys3NlSQVFxcrLy9PXq9XGzdu1OnTp1VTU6OKigpWaQAAgCPmovPMM8+oq6tLd91116hjq1at0uDgoCorKxUMBlVYWKimpialpKQ4mS1btighIUGLFi3S4OCgZs+ere3btys+Pt7J7Ny5U1VVVc6ns8rKylRfX+8cj4+P1+7du1VZWakZM2YoKSlJ5eXl2rRpU6y3AwAADHZZ36NzreN7dK4svkcHAPBh+FC+RwcAAOBqR9EBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABgr5qLzhz/8Qf/4j/+oiRMnavz48frbv/1btbe3O8dt29batWvl8XiUlJSkWbNm6ciRI1GvEQ6HtXLlSqWnpys5OVllZWU6fvx4VCYYDMrr9cqyLFmWJa/XqzNnzkRlurq6tHDhQiUnJys9PV1VVVWKRCKx3hIAADBUTEUnGAxqxowZGjdunPbu3auXXnpJDz30kD7+8Y87mQ0bNmjz5s2qr6/XoUOH5Ha7NXfuXPX39zsZn8+nhoYG+f1+tba26uzZsyotLdXw8LCTKS8vV0dHhxobG9XY2KiOjg55vV7n+PDwsBYsWKCBgQG1trbK7/dr165dqq6uvozpAAAAJomzbdu+2PD999+v//mf/9FvfvOb8x63bVsej0c+n0+rV6+W9M7qTWZmptavX68VK1YoFApp0qRJ2rFjhxYvXixJOnHihLKysrRnzx6VlJTo6NGjysvLU1tbmwoLCyVJbW1tKioq0ssvv6zc3Fzt3btXpaWl6u7ulsfjkST5/X4tXbpUvb29Sk1Nfd/76evrk2VZCoVCF5WP1c33777ir3k1e33dgrEeAgDgOhDL7++YVnSefvpp3X777frKV76ijIwM3Xbbbdq2bZtz/NixYwoEAiouLnb2uVwuzZw5U/v27ZMktbe3a2hoKCrj8XiUn5/vZPbv3y/LspySI0nTpk2TZVlRmfz8fKfkSFJJSYnC4XDUf6X9pXA4rL6+vqgNAACYK6ai89prr+mRRx5RTk6OfvWrX+nuu+9WVVWVfvrTn0qSAoGAJCkzMzPqvMzMTOdYIBBQYmKi0tLSLpjJyMgYdf2MjIyozMjrpKWlKTEx0cmMVFdX57znx7IsZWVlxXL7AADgGhNT0Tl37pw+85nPqLa2VrfddptWrFihiooKPfLII1G5uLi4qJ9t2x61b6SRmfPlLyXzl9asWaNQKORs3d3dFxwTAAC4tsVUdCZPnqy8vLyofVOmTFFXV5ckye12S9KoFZXe3l5n9cXtdisSiSgYDF4wc/LkyVHXP3XqVFRm5HWCwaCGhoZGrfS8y+VyKTU1NWoDAADmiqnozJgxQ6+88krUvt///ve66aabJEnZ2dlyu91qbm52jkciEbW0tGj69OmSpIKCAo0bNy4q09PTo87OTidTVFSkUCikgwcPOpkDBw4oFApFZTo7O9XT0+Nkmpqa5HK5VFBQEMttAQAAQyXEEv7nf/5nTZ8+XbW1tVq0aJEOHjyoRx99VI8++qikd/4ryefzqba2Vjk5OcrJyVFtba3Gjx+v8vJySZJlWVq2bJmqq6s1ceJETZgwQTU1NZo6darmzJkj6Z1Vonnz5qmiokJbt26VJC1fvlylpaXKzc2VJBUXFysvL09er1cbN27U6dOnVVNTo4qKClZqAACApBiLzmc/+1k1NDRozZo1+u53v6vs7Gw9/PDD+vrXv+5kVq1apcHBQVVWVioYDKqwsFBNTU1KSUlxMlu2bFFCQoIWLVqkwcFBzZ49W9u3b1d8fLyT2blzp6qqqpxPZ5WVlam+vt45Hh8fr927d6uyslIzZsxQUlKSysvLtWnTpkueDAAAYJaYvkfHNHyPzpXF9+gAAD4MH9j36AAAAFxLKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGPFVHTWrl2ruLi4qM3tdjvHbdvW2rVr5fF4lJSUpFmzZunIkSNRrxEOh7Vy5Uqlp6crOTlZZWVlOn78eFQmGAzK6/XKsixZliWv16szZ85EZbq6urRw4UIlJycrPT1dVVVVikQiMd4+AAAwWcwrOp/+9KfV09PjbIcPH3aObdiwQZs3b1Z9fb0OHTokt9utuXPnqr+/38n4fD41NDTI7/ertbVVZ8+eVWlpqYaHh51MeXm5Ojo61NjYqMbGRnV0dMjr9TrHh4eHtWDBAg0MDKi1tVV+v1+7du1SdXX1pc4DAAAwUELMJyQkRK3ivMu2bT388MN64IEHdOedd0qSnnjiCWVmZuqpp57SihUrFAqF9Nhjj2nHjh2aM2eOJOnJJ59UVlaWnnnmGZWUlOjo0aNqbGxUW1ubCgsLJUnbtm1TUVGRXnnlFeXm5qqpqUkvvfSSuru75fF4JEkPPfSQli5dqu9973tKTU295AkBAADmiHlF59VXX5XH41F2dra++tWv6rXXXpMkHTt2TIFAQMXFxU7W5XJp5syZ2rdvnySpvb1dQ0NDURmPx6P8/Hwns3//flmW5ZQcSZo2bZosy4rK5OfnOyVHkkpKShQOh9Xe3v6eYw+Hw+rr64vaAACAuWIqOoWFhfrpT3+qX/3qV9q2bZsCgYCmT5+uN998U4FAQJKUmZkZdU5mZqZzLBAIKDExUWlpaRfMZGRkjLp2RkZGVGbkddLS0pSYmOhkzqeurs55349lWcrKyorl9gEAwDUmpqIzf/58ffnLX9bUqVM1Z84c7d69W9I7/0X1rri4uKhzbNsetW+kkZnz5S8lM9KaNWsUCoWcrbu7+4LjAgAA17bL+nh5cnKypk6dqldffdV5387IFZXe3l5n9cXtdisSiSgYDF4wc/LkyVHXOnXqVFRm5HWCwaCGhoZGrfT8JZfLpdTU1KgNAACY67KKTjgc1tGjRzV58mRlZ2fL7XarubnZOR6JRNTS0qLp06dLkgoKCjRu3LioTE9Pjzo7O51MUVGRQqGQDh486GQOHDigUCgUlens7FRPT4+TaWpqksvlUkFBweXcEgAAMEhMn7qqqanRwoULdeONN6q3t1f//u//rr6+Pi1ZskRxcXHy+Xyqra1VTk6OcnJyVFtbq/Hjx6u8vFySZFmWli1bpurqak2cOFETJkxQTU2N819hkjRlyhTNmzdPFRUV2rp1qyRp+fLlKi0tVW5uriSpuLhYeXl58nq92rhxo06fPq2amhpVVFSwSgMAABwxFZ3jx4/ra1/7mv74xz9q0qRJmjZtmtra2nTTTTdJklatWqXBwUFVVlYqGAyqsLBQTU1NSklJcV5jy5YtSkhI0KJFizQ4OKjZs2dr+/btio+PdzI7d+5UVVWV8+mssrIy1dfXO8fj4+O1e/duVVZWasaMGUpKSlJ5ebk2bdp0WZMBAADMEmfbtj3WgxgrfX19sixLoVDoA1kJuvn+3Vf8Na9mr69bMNZDAABcB2L5/c3fugIAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjHVZRaeurk5xcXHy+XzOPtu2tXbtWnk8HiUlJWnWrFk6cuRI1HnhcFgrV65Uenq6kpOTVVZWpuPHj0dlgsGgvF6vLMuSZVnyer06c+ZMVKarq0sLFy5UcnKy0tPTVVVVpUgkcjm3BAAADHLJRefQoUN69NFHdcstt0Tt37BhgzZv3qz6+nodOnRIbrdbc+fOVX9/v5Px+XxqaGiQ3+9Xa2urzp49q9LSUg0PDzuZ8vJydXR0qLGxUY2Njero6JDX63WODw8Pa8GCBRoYGFBra6v8fr927dql6urqS70lAABgmEsqOmfPntXXv/51bdu2TWlpac5+27b18MMP64EHHtCdd96p/Px8PfHEE3rrrbf01FNPSZJCoZAee+wxPfTQQ5ozZ45uu+02Pfnkkzp8+LCeeeYZSdLRo0fV2Nion/zkJyoqKlJRUZG2bdumX/7yl3rllVckSU1NTXrppZf05JNP6rbbbtOcOXP00EMPadu2berr67vceQEAAAa4pKJzzz33aMGCBZozZ07U/mPHjikQCKi4uNjZ53K5NHPmTO3bt0+S1N7erqGhoaiMx+NRfn6+k9m/f78sy1JhYaGTmTZtmizLisrk5+fL4/E4mZKSEoXDYbW3t5933OFwWH19fVEbAAAwV0KsJ/j9frW3t+uFF14YdSwQCEiSMjMzo/ZnZmbqjTfecDKJiYlRK0HvZt49PxAIKCMjY9TrZ2RkRGVGXictLU2JiYlOZqS6ujo9+OCDF3ObAADAADGt6HR3d+uf/umftHPnTn30ox99z1xcXFzUz7Ztj9o30sjM+fKXkvlLa9asUSgUcrbu7u4LjgkAAFzbYio67e3t6u3tVUFBgRISEpSQkKCWlhb9x3/8hxISEpwVlpErKr29vc4xt9utSCSiYDB4wczJkydHXf/UqVNRmZHXCQaDGhoaGrXS8y6Xy6XU1NSoDQAAmCumojN79mwdPnxYHR0dznb77bfr61//ujo6OvSJT3xCbrdbzc3NzjmRSEQtLS2aPn26JKmgoEDjxo2LyvT09Kizs9PJFBUVKRQK6eDBg07mwIEDCoVCUZnOzk719PQ4maamJrlcLhUUFFzCVAAAANPE9B6dlJQU5efnR+1LTk7WxIkTnf0+n0+1tbXKyclRTk6OamtrNX78eJWXl0uSLMvSsmXLVF1drYkTJ2rChAmqqanR1KlTnTc3T5kyRfPmzVNFRYW2bt0qSVq+fLlKS0uVm5srSSouLlZeXp68Xq82btyo06dPq6amRhUVFazUAAAASZfwZuT3s2rVKg0ODqqyslLBYFCFhYVqampSSkqKk9myZYsSEhK0aNEiDQ4Oavbs2dq+fbvi4+OdzM6dO1VVVeV8OqusrEz19fXO8fj4eO3evVuVlZWaMWOGkpKSVF5erk2bNl3pWwIAANeoONu27bEexFjp6+uTZVkKhUIfyCrQzffvvuKveTV7fd2CsR4CAOA6EMvvb/7WFQAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgrJiKziOPPKJbbrlFqampSk1NVVFRkfbu3esct21ba9eulcfjUVJSkmbNmqUjR45EvUY4HNbKlSuVnp6u5ORklZWV6fjx41GZYDAor9cry7JkWZa8Xq/OnDkTlenq6tLChQuVnJys9PR0VVVVKRKJxHj7AADAZDEVnRtuuEHr1q3TCy+8oBdeeEF33HGHvvjFLzplZsOGDdq8ebPq6+t16NAhud1uzZ07V/39/c5r+Hw+NTQ0yO/3q7W1VWfPnlVpaamGh4edTHl5uTo6OtTY2KjGxkZ1dHTI6/U6x4eHh7VgwQINDAyotbVVfr9fu3btUnV19eXOBwAAMEicbdv25bzAhAkTtHHjRt11113yeDzy+XxavXq1pHdWbzIzM7V+/XqtWLFCoVBIkyZN0o4dO7R48WJJ0okTJ5SVlaU9e/aopKRER48eVV5entra2lRYWChJamtrU1FRkV5++WXl5uZq7969Ki0tVXd3tzwejyTJ7/dr6dKl6u3tVWpq6kWNva+vT5ZlKRQKXfQ5sbj5/t1X/DWvZq+vWzDWQwAAXAdi+f19ye/RGR4elt/v18DAgIqKinTs2DEFAgEVFxc7GZfLpZkzZ2rfvn2SpPb2dg0NDUVlPB6P8vPzncz+/ftlWZZTciRp2rRpsiwrKpOfn++UHEkqKSlROBxWe3v7e445HA6rr68vagMAAOaKuegcPnxYH/vYx+RyuXT33XeroaFBeXl5CgQCkqTMzMyofGZmpnMsEAgoMTFRaWlpF8xkZGSMum5GRkZUZuR10tLSlJiY6GTOp66uznnfj2VZysrKivHuAQDAtSTmopObm6uOjg61tbXpm9/8ppYsWaKXXnrJOR4XFxeVt2171L6RRmbOl7+UzEhr1qxRKBRytu7u7guOCwAAXNtiLjqJiYn65Cc/qdtvv111dXW69dZb9f3vf19ut1uSRq2o9Pb2OqsvbrdbkUhEwWDwgpmTJ0+Ouu6pU6eiMiOvEwwGNTQ0NGql5y+5XC7nE2PvbgAAwFyX/T06tm0rHA4rOztbbrdbzc3NzrFIJKKWlhZNnz5dklRQUKBx48ZFZXp6etTZ2elkioqKFAqFdPDgQSdz4MABhUKhqExnZ6d6enqcTFNTk1wulwoKCi73lgAAgCESYgl/+9vf1vz585WVlaX+/n75/X49//zzamxsVFxcnHw+n2pra5WTk6OcnBzV1tZq/PjxKi8vlyRZlqVly5apurpaEydO1IQJE1RTU6OpU6dqzpw5kqQpU6Zo3rx5qqio0NatWyVJy5cvV2lpqXJzcyVJxcXFysvLk9fr1caNG3X69GnV1NSooqKCVRoAAOCIqeicPHlSXq9XPT09sixLt9xyixobGzV37lxJ0qpVqzQ4OKjKykoFg0EVFhaqqalJKSkpzmts2bJFCQkJWrRokQYHBzV79mxt375d8fHxTmbnzp2qqqpyPp1VVlam+vp653h8fLx2796tyspKzZgxQ0lJSSovL9emTZsuazIAAIBZLvt7dK5lfI/OlcX36AAAPgwfyvfoAAAAXO0oOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY8VUdOrq6vTZz35WKSkpysjI0Je+9CW98sorURnbtrV27Vp5PB4lJSVp1qxZOnLkSFQmHA5r5cqVSk9PV3JyssrKynT8+PGoTDAYlNfrlWVZsixLXq9XZ86cicp0dXVp4cKFSk5OVnp6uqqqqhSJRGK5JQAAYLCYik5LS4vuuecetbW1qbm5WW+//baKi4s1MDDgZDZs2KDNmzervr5ehw4dktvt1ty5c9Xf3+9kfD6fGhoa5Pf71draqrNnz6q0tFTDw8NOpry8XB0dHWpsbFRjY6M6Ojrk9Xqd48PDw1qwYIEGBgbU2toqv9+vXbt2qbq6+nLmAwAAGCTOtm37Uk8+deqUMjIy1NLSoi984QuybVsej0c+n0+rV6+W9M7qTWZmptavX68VK1YoFApp0qRJ2rFjhxYvXixJOnHihLKysrRnzx6VlJTo6NGjysvLU1tbmwoLCyVJbW1tKioq0ssvv6zc3Fzt3btXpaWl6u7ulsfjkST5/X4tXbpUvb29Sk1Nfd/x9/X1ybIshUKhi8rH6ub7d1/x17yavb5uwVgPAQBwHYjl9/dlvUcnFApJkiZMmCBJOnbsmAKBgIqLi52My+XSzJkztW/fPklSe3u7hoaGojIej0f5+flOZv/+/bIsyyk5kjRt2jRZlhWVyc/Pd0qOJJWUlCgcDqu9vf284w2Hw+rr64vaAACAuS656Ni2rfvuu0+f+9znlJ+fL0kKBAKSpMzMzKhsZmamcywQCCgxMVFpaWkXzGRkZIy6ZkZGRlRm5HXS0tKUmJjoZEaqq6tz3vNjWZaysrJivW0AAHANueSic++99+p3v/ud/vM//3PUsbi4uKifbdsetW+kkZnz5S8l85fWrFmjUCjkbN3d3RccEwAAuLZdUtFZuXKlnn76aT333HO64YYbnP1ut1uSRq2o9Pb2OqsvbrdbkUhEwWDwgpmTJ0+Ouu6pU6eiMiOvEwwGNTQ0NGql510ul0upqalRGwAAMFdMRce2bd177736+c9/rmeffVbZ2dlRx7Ozs+V2u9Xc3Ozsi0Qiamlp0fTp0yVJBQUFGjduXFSmp6dHnZ2dTqaoqEihUEgHDx50MgcOHFAoFIrKdHZ2qqenx8k0NTXJ5XKpoKAgltsCAACGSoglfM899+ipp57Sf/3XfyklJcVZUbEsS0lJSYqLi5PP51Ntba1ycnKUk5Oj2tpajR8/XuXl5U522bJlqq6u1sSJEzVhwgTV1NRo6tSpmjNnjiRpypQpmjdvnioqKrR161ZJ0vLly1VaWqrc3FxJUnFxsfLy8uT1erVx40adPn1aNTU1qqioYKUGAABIivHj5e/13pfHH39cS5culfTOqs+DDz6orVu3KhgMqrCwUD/84Q+dNyxL0p/+9Cd961vf0lNPPaXBwUHNnj1bP/rRj6LeHHz69GlVVVXp6aefliSVlZWpvr5eH//4x51MV1eXKisr9eyzzyopKUnl5eXatGmTXC7XRd0PHy83Ax9rB4DrSyy/vy/re3SudRQdM1B0AOD68qF9jw4AAMDVjKIDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADBWzEXn17/+tRYuXCiPx6O4uDj94he/iDpu27bWrl0rj8ejpKQkzZo1S0eOHInKhMNhrVy5Uunp6UpOTlZZWZmOHz8elQkGg/J6vbIsS5Zlyev16syZM1GZrq4uLVy4UMnJyUpPT1dVVZUikUistwQAAAwVc9EZGBjQrbfeqvr6+vMe37BhgzZv3qz6+nodOnRIbrdbc+fOVX9/v5Px+XxqaGiQ3+9Xa2urzp49q9LSUg0PDzuZ8vJydXR0qLGxUY2Njero6JDX63WODw8Pa8GCBRoYGFBra6v8fr927dql6urqWG8JAAAYKs62bfuST46LU0NDg770pS9Jemc1x+PxyOfzafXq1ZLeWb3JzMzU+vXrtWLFCoVCIU2aNEk7duzQ4sWLJUknTpxQVlaW9uzZo5KSEh09elR5eXlqa2tTYWGhJKmtrU1FRUV6+eWXlZubq71796q0tFTd3d3yeDySJL/fr6VLl6q3t1epqanvO/6+vj5ZlqVQKHRR+VjdfP/uK/6aGO31dQvGeggAgA9RLL+/r+h7dI4dO6ZAIKDi4mJnn8vl0syZM7Vv3z5JUnt7u4aGhqIyHo9H+fn5Tmb//v2yLMspOZI0bdo0WZYVlcnPz3dKjiSVlJQoHA6rvb39vOMLh8Pq6+uL2gAAgLmuaNEJBAKSpMzMzKj9mZmZzrFAIKDExESlpaVdMJORkTHq9TMyMqIyI6+TlpamxMREJzNSXV2d854fy7KUlZV1CXcJAACuFR/Ip67i4uKifrZte9S+kUZmzpe/lMxfWrNmjUKhkLN1d3dfcEwAAODadkWLjtvtlqRRKyq9vb3O6ovb7VYkElEwGLxg5uTJk6Ne/9SpU1GZkdcJBoMaGhoatdLzLpfLpdTU1KgNAACY64oWnezsbLndbjU3Nzv7IpGIWlpaNH36dElSQUGBxo0bF5Xp6elRZ2enkykqKlIoFNLBgwedzIEDBxQKhaIynZ2d6unpcTJNTU1yuVwqKCi4krcFAACuUQmxnnD27Fn93//9n/PzsWPH1NHRoQkTJujGG2+Uz+dTbW2tcnJylJOTo9raWo0fP17l5eWSJMuytGzZMlVXV2vixImaMGGCampqNHXqVM2ZM0eSNGXKFM2bN08VFRXaunWrJGn58uUqLS1Vbm6uJKm4uFh5eXnyer3auHGjTp8+rZqaGlVUVLBSAwAAJF1C0XnhhRf093//987P9913nyRpyZIl2r59u1atWqXBwUFVVlYqGAyqsLBQTU1NSklJcc7ZsmWLEhIStGjRIg0ODmr27Nnavn274uPjnczOnTtVVVXlfDqrrKws6rt74uPjtXv3blVWVmrGjBlKSkpSeXm5Nm3aFPssAAAAI13W9+hc6/geHTPwPToAcH0Zs+/RAQAAuJpQdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjJYz1AIDLdfP9u8fkuq+vWzAm1wUAXDxWdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi79eDlwi/mo6AFz9rvkVnR/96EfKzs7WRz/6URUUFOg3v/nNWA8JAABcJa7povOzn/1MPp9PDzzwgF588UV9/vOf1/z589XV1TXWQwMAAFeBa7robN68WcuWLdM3vvENTZkyRQ8//LCysrL0yCOPjPXQAADAVeCafY9OJBJRe3u77r///qj9xcXF2rdv33nPCYfDCofDzs+hUEiS1NfX94GM8Vz4rQ/kdXF9u/Gf/9+YXLfzwZIxuS4AjPTu723btt83e80WnT/+8Y8aHh5WZmZm1P7MzEwFAoHznlNXV6cHH3xw1P6srKwPZIyASayHx3oEABCtv79flmVdMHPNFp13xcXFRf1s2/aofe9as2aN7rvvPufnc+fO6fTp05o4ceJ7nhOrvr4+ZWVlqbu7W6mpqVfkNU3HnMWG+YoN8xU75iw2zFfsLnfObNtWf3+/PB7P+2av2aKTnp6u+Pj4Uas3vb29o1Z53uVyueRyuaL2ffzjH/9AxpeamsoDHyPmLDbMV2yYr9gxZ7FhvmJ3OXP2fis577pm34ycmJiogoICNTc3R+1vbm7W9OnTx2hUAADganLNruhI0n333Sev16vbb79dRUVFevTRR9XV1aW77757rIcGAACuAtd00Vm8eLHefPNNffe731VPT4/y8/O1Z88e3XTTTWM2JpfLpe985zuj/osM7405iw3zFRvmK3bMWWyYr9h9mHMWZ1/MZ7MAAACuQdfse3QAAADeD0UHAAAYi6IDAACMRdEBAADGouhcYT/60Y+UnZ2tj370oyooKNBvfvObsR7SVWHt2rWKi4uL2txut3Pctm2tXbtWHo9HSUlJmjVrlo4cOTKGI/5w/frXv9bChQvl8XgUFxenX/ziF1HHL2Z+wuGwVq5cqfT0dCUnJ6usrEzHjx//EO/iw/V+c7Z06dJRz9y0adOiMtfTnNXV1emzn/2sUlJSlJGRoS996Ut65ZVXojI8Z392MfPFM/ZnjzzyiG655RbnCwCLioq0d+9e5/hYPlsUnSvoZz/7mXw+nx544AG9+OKL+vznP6/58+erq6trrId2Vfj0pz+tnp4eZzt8+LBzbMOGDdq8ebPq6+t16NAhud1uzZ07V/39/WM44g/PwMCAbr31VtXX15/3+MXMj8/nU0NDg/x+v1pbW3X27FmVlpZqeHj4w7qND9X7zZkkzZs3L+qZ27NnT9Tx62nOWlpadM8996itrU3Nzc16++23VVxcrIGBASfDc/ZnFzNfEs/Yu2644QatW7dOL7zwgl544QXdcccd+uIXv+iUmTF9tmxcMX/3d39n33333VH7PvWpT9n333//GI3o6vGd73zHvvXWW8977Ny5c7bb7bbXrVvn7PvTn/5kW5Zl//jHP/6QRnj1kGQ3NDQ4P1/M/Jw5c8YeN26c7ff7ncwf/vAH+yMf+Yjd2Nj4oY19rIycM9u27SVLlthf/OIX3/Oc633Oent7bUl2S0uLbds8Z+9n5HzZNs/Y+0lLS7N/8pOfjPmzxYrOFRKJRNTe3q7i4uKo/cXFxdq3b98Yjerq8uqrr8rj8Sg7O1tf/epX9dprr0mSjh07pkAgEDV3LpdLM2fOZO50cfPT3t6uoaGhqIzH41F+fv51PYfPP/+8MjIy9Dd/8zeqqKhQb2+vc+x6n7NQKCRJmjBhgiSes/czcr7exTM22vDwsPx+vwYGBlRUVDTmzxZF5wr54x//qOHh4VF/UDQzM3PUHx69HhUWFuqnP/2pfvWrX2nbtm0KBAKaPn263nzzTWd+mLvzu5j5CQQCSkxMVFpa2ntmrjfz58/Xzp079eyzz+qhhx7SoUOHdMcddygcDku6vufMtm3dd999+tznPqf8/HxJPGcXcr75knjGRjp8+LA+9rGPyeVy6e6771ZDQ4Py8vLG/Nm6pv8ExNUoLi4u6mfbtkftux7Nnz/f+ffUqVNVVFSkv/7rv9YTTzzhvHmPubuwS5mf63kOFy9e7Pw7Pz9ft99+u2666Sbt3r1bd95553uedz3M2b333qvf/e53am1tHXWM52y095ovnrFoubm56ujo0JkzZ7Rr1y4tWbJELS0tzvGxerZY0blC0tPTFR8fP6p59vb2jmqxkJKTkzV16lS9+uqrzqevmLvzu5j5cbvdikQiCgaD75m53k2ePFk33XSTXn31VUnX75ytXLlSTz/9tJ577jndcMMNzn6es/N7r/k6n+v9GUtMTNQnP/lJ3X777aqrq9Ott96q73//+2P+bFF0rpDExEQVFBSoubk5an9zc7OmT58+RqO6eoXDYR09elSTJ09Wdna23G531NxFIhG1tLQwd9JFzU9BQYHGjRsXlenp6VFnZydz+P9788031d3drcmTJ0u6/ubMtm3de++9+vnPf65nn31W2dnZUcd5zqK933ydz/X+jI1k27bC4fDYP1uX9VZmRPH7/fa4cePsxx57zH7ppZdsn89nJycn26+//vpYD23MVVdX288//7z92muv2W1tbXZpaamdkpLizM26detsy7Lsn//85/bhw4ftr33ta/bkyZPtvr6+MR75h6O/v99+8cUX7RdffNGWZG/evNl+8cUX7TfeeMO27Yubn7vvvtu+4YYb7Geeecb+7W9/a99xxx32rbfear/99ttjdVsfqAvNWX9/v11dXW3v27fPPnbsmP3cc8/ZRUVF9l/91V9dt3P2zW9+07Ysy37++eftnp4eZ3vrrbecDM/Zn73ffPGMRVuzZo3961//2j527Jj9u9/9zv72t79tf+QjH7Gbmpps2x7bZ4uic4X98Ic/tG+66SY7MTHR/sxnPhP1UcTr2eLFi+3Jkyfb48aNsz0ej33nnXfaR44ccY6fO3fO/s53vmO73W7b5XLZX/jCF+zDhw+P4Yg/XM8995wtadS2ZMkS27Yvbn4GBwfte++9154wYYKdlJRkl5aW2l1dXWNwNx+OC83ZW2+9ZRcXF9uTJk2yx40bZ9944432kiVLRs3H9TRn55srSfbjjz/uZHjO/uz95otnLNpdd93l/O6bNGmSPXv2bKfk2PbYPltxtm3bl7cmBAAAcHXiPToAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGOv/A7/wiFhaBGrLAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(sent_lens, bins=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c6798a81",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "55"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_seq_len = int(np.percentile(sent_lens, 95))\n",
    "output_seq_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "2fc653fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "max_tokens = 68000\n",
    "\n",
    "text_vectorizer = tf.keras.layers.TextVectorization(\n",
    "    max_tokens = max_tokens,\n",
    "    output_sequence_length = output_seq_len,\n",
    "    pad_to_max_tokens=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b41e514e",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_vectorizer.adapt(train_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "305b8299",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text:\n",
      " `` the coach program '' was developed in australia as target driven educational telephone delivered intervention to support people with different chronic conditions .\n",
      "\n",
      "Length of text: 166\n",
      "\n",
      "Vectorized text: [[   2 7106  256   10  481    5 1696   25  492 4386  904 1032  785   38\n",
      "     6  279  519    7  197  244  458    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0]]\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "target_sentence = random.choice(train_sentences)\n",
    "print(f\"Text:\\n {target_sentence}\")\n",
    "print(f\"\\nLength of text: {len(target_sentence)}\")\n",
    "print(f\"\\nVectorized text: {text_vectorizer([target_sentence])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e3a7ca36",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of words in vocab: 64841\n",
      "Most common words in the vocab: ['', '[UNK]', 'the', 'and', 'of']\n",
      "Least common words in the vocab: ['aainduced', 'aaigroup', 'aachener', 'aachen', 'aaacp']\n"
     ]
    }
   ],
   "source": [
    "rct_20k_text_vocab = text_vectorizer.get_vocabulary()\n",
    "print(f\"Number of words in vocab: {len(rct_20k_text_vocab)}\")\n",
    "print(f\"Most common words in the vocab: {rct_20k_text_vocab[:5]}\")\n",
    "print(f\"Least common words in the vocab: {rct_20k_text_vocab[-5:]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "dc0aff54",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'text_vectorization',\n",
       " 'trainable': True,\n",
       " 'batch_input_shape': (None,),\n",
       " 'dtype': 'string',\n",
       " 'max_tokens': 68000,\n",
       " 'standardize': 'lower_and_strip_punctuation',\n",
       " 'split': 'whitespace',\n",
       " 'ngrams': None,\n",
       " 'output_mode': 'int',\n",
       " 'output_sequence_length': 55,\n",
       " 'pad_to_max_tokens': True}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_vectorizer.get_config()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "3f42c295",
   "metadata": {},
   "outputs": [],
   "source": [
    "token_embed = tf.keras.layers.Embedding(\n",
    "    input_dim = len(rct_20k_text_vocab),\n",
    "    output_dim = 128,\n",
    "    mask_zero = True,\n",
    "    name='token_embedding'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "65029bfa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence before vectorization:\n",
      " `` the coach program '' was developed in australia as target driven educational telephone delivered intervention to support people with different chronic conditions .\n",
      "\n",
      "Sentence after vectorization (before embedding):\n",
      " [[   2 7106  256   10  481    5 1696   25  492 4386  904 1032  785   38\n",
      "     6  279  519    7  197  244  458    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0]]\n",
      "\n",
      "Vectorized sentence shape: (1, 55)\n",
      "\n",
      "Sentence after embedding:\n",
      " [[[ 0.00273906 -0.03621719 -0.02919917 ... -0.03867118 -0.0027074\n",
      "   -0.04348738]\n",
      "  [-0.00578468 -0.00376404  0.03207007 ...  0.01953218  0.03462989\n",
      "   -0.00949457]\n",
      "  [ 0.03772724 -0.04546225  0.04502401 ...  0.00348092  0.00975914\n",
      "    0.03382533]\n",
      "  ...\n",
      "  [-0.0343977  -0.01226209  0.01615605 ... -0.03026294  0.01575447\n",
      "    0.01626864]\n",
      "  [-0.0343977  -0.01226209  0.01615605 ... -0.03026294  0.01575447\n",
      "    0.01626864]\n",
      "  [-0.0343977  -0.01226209  0.01615605 ... -0.03026294  0.01575447\n",
      "    0.01626864]]]\n",
      "\n",
      "Embedded sentence shape: (1, 55, 128)\n"
     ]
    }
   ],
   "source": [
    "print(f\"Sentence before vectorization:\\n {target_sentence}\\n\")\n",
    "vectorized_sentence = text_vectorizer([target_sentence])\n",
    "print(f\"Sentence after vectorization (before embedding):\\n {vectorized_sentence}\\n\")\n",
    "print(f\"Vectorized sentence shape: {vectorized_sentence.shape}\\n\")\n",
    "embedded_sentence = token_embed(vectorized_sentence)\n",
    "print(f\"Sentence after embedding:\\n {embedded_sentence}\\n\")\n",
    "print(f\"Embedded sentence shape: {embedded_sentence.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "9d6e2a57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<TensorSliceDataset shapes: ((), (5,)), types: (tf.string, tf.float64)>,\n",
       " <TensorSliceDataset shapes: ((), (5,)), types: (tf.string, tf.float64)>,\n",
       " <TensorSliceDataset shapes: ((), (5,)), types: (tf.string, tf.float64)>)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset = tf.data.Dataset.from_tensor_slices((train_sentences, train_labels_one_hot))\n",
    "val_dataset = tf.data.Dataset.from_tensor_slices((val_sentences, val_labels_one_hot))\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test_sentences, test_labels_one_hot))\n",
    "\n",
    "train_dataset, val_dataset, test_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "723b856c",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = train_dataset.batch(32).prefetch(tf.data.AUTOTUNE)\n",
    "val_dataset = val_dataset.batch(32).prefetch(tf.data.AUTOTUNE)\n",
    "test_dataset = test_dataset.batch(32).prefetch(tf.data.AUTOTUNE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "edb150fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Model-1\n",
    "inputs = layers.Input(shape=(1,), dtype=tf.string)\n",
    "x = text_vectorizer(inputs)\n",
    "x = token_embed(x)\n",
    "x = layers.Conv1D(64, 5, activation='relu')(x)\n",
    "x = layers.GlobalAveragePooling1D()(x)\n",
    "outputs = layers.Dense(num_classes, activation='softmax')(x)\n",
    "model_1 = tf.keras.Model(inputs, outputs, name='model_1_Conv1D_Token')\n",
    "\n",
    "model_1.compile(\n",
    "    optimizer=tf.keras.optimizers.Adam(),\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['accuracy']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "aeca6176",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1_Conv1D_Token\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 1)]               0         \n",
      "_________________________________________________________________\n",
      "text_vectorization (TextVect (None, 55)                0         \n",
      "_________________________________________________________________\n",
      "token_embedding (Embedding)  (None, 55, 128)           8299648   \n",
      "_________________________________________________________________\n",
      "conv1d (Conv1D)              (None, 51, 64)            41024     \n",
      "_________________________________________________________________\n",
      "global_average_pooling1d (Gl (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 5)                 325       \n",
      "=================================================================\n",
      "Total params: 8,340,997\n",
      "Trainable params: 8,340,997\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "ef15ddc8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "562/562 [==============================] - 15s 21ms/step - loss: 0.9431 - accuracy: 0.6264 - val_loss: 0.7237 - val_accuracy: 0.7234\n",
      "Epoch 2/5\n",
      "562/562 [==============================] - 11s 20ms/step - loss: 0.6868 - accuracy: 0.7456 - val_loss: 0.6623 - val_accuracy: 0.7603\n",
      "Epoch 3/5\n",
      "562/562 [==============================] - 11s 20ms/step - loss: 0.6389 - accuracy: 0.7658 - val_loss: 0.6166 - val_accuracy: 0.7716\n",
      "Epoch 4/5\n",
      "562/562 [==============================] - 11s 20ms/step - loss: 0.6078 - accuracy: 0.7794 - val_loss: 0.5948 - val_accuracy: 0.7839\n",
      "Epoch 5/5\n",
      "562/562 [==============================] - 11s 20ms/step - loss: 0.6026 - accuracy: 0.7879 - val_loss: 0.5732 - val_accuracy: 0.7932\n"
     ]
    }
   ],
   "source": [
    "model_1_history = model_1.fit(\n",
    "    train_dataset,\n",
    "    steps_per_epoch=int(0.1 * len(train_dataset)),\n",
    "    epochs=5,\n",
    "    validation_data=val_dataset,\n",
    "    validation_steps=int(0.1 *len(val_dataset))\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "5a468c4f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "945/945 [==============================] - 5s 5ms/step - loss: 0.5717 - accuracy: 0.7980\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.5717340111732483, 0.7979610562324524]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_1.evaluate(val_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "d4c7fc59",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5.2922732e-01, 1.7782696e-01, 5.3315051e-02, 1.8499327e-01,\n",
       "        5.4637458e-02],\n",
       "       [4.8181516e-01, 2.5811255e-01, 2.8693138e-02, 1.9117597e-01,\n",
       "        4.0203225e-02],\n",
       "       [1.9519117e-01, 6.8503004e-03, 2.0161176e-03, 7.9589337e-01,\n",
       "        4.8952639e-05],\n",
       "       ...,\n",
       "       [2.5659832e-05, 6.3063850e-04, 2.9625401e-03, 2.8507211e-05,\n",
       "        9.9635267e-01],\n",
       "       [8.4832378e-02, 2.5359359e-01, 2.2257721e-01, 7.9612084e-02,\n",
       "        3.5938475e-01],\n",
       "       [1.7333665e-01, 6.5409374e-01, 1.4291541e-01, 1.3787505e-02,\n",
       "        1.5866725e-02]], dtype=float32)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_1_pred_probs = model_1.predict(val_dataset)\n",
    "model_1_pred_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "d80d6937",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(30212,), dtype=int64, numpy=array([0, 0, 3, ..., 4, 4, 1], dtype=int64)>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_1_preds =tf.argmax(model_1_pred_probs, axis=1)\n",
    "model_1_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "566d57ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 79.79610750695088,\n",
       " 'precision': 0.7961424985247666,\n",
       " 'recall': 0.7979610750695088,\n",
       " 'f1-score': 0.7945987966997599}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_1_results = calculate_results(val_labels_encoded, model_1_preds)\n",
    "model_1_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "8eb90119",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 72.1832384482987,\n",
       " 'precision': 0.7186466952323352,\n",
       " 'recall': 0.7218323844829869,\n",
       " 'f1-score': 0.6989250353450294}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseline_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "8944ab6a",
   "metadata": {},
   "outputs": [
    {
     "ename": "UnknownError",
     "evalue": "Failed to WriteFile: C:\\Users\\Gurpr\\AppData\\Local\\Temp\\tfhub_modules\\063d866c06683311b44b4992fd46003be952409c.87dc21ef12df4d4997e0f0a03b89577a.tmp\\variables\\variables.data-00000-of-00001 : There is not enough space on the disk.\r\n; operation in progress",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mUnknownError\u001b[0m                              Traceback (most recent call last)",
      "Input \u001b[1;32mIn [45]\u001b[0m, in \u001b[0;36m<cell line: 3>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtensorflow_hub\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mhub\u001b[39;00m\n\u001b[1;32m----> 3\u001b[0m uni_sentence_encoder \u001b[38;5;241m=\u001b[39m \u001b[43mhub\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mKerasLayer\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhttps://tfhub.dev/google/universal-sentence-encoder/4\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m      4\u001b[0m \u001b[43m                                     \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43muni_sentence_encoder\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\tf-proper\\lib\\site-packages\\tensorflow_hub\\keras_layer.py:153\u001b[0m, in \u001b[0;36mKerasLayer.__init__\u001b[1;34m(self, handle, trainable, arguments, _sentinel, tags, signature, signature_outputs_as_dict, output_key, output_shape, load_options, **kwargs)\u001b[0m\n\u001b[0;32m    149\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_output_shape \u001b[38;5;241m=\u001b[39m data_structures\u001b[38;5;241m.\u001b[39mNoDependency(\n\u001b[0;32m    150\u001b[0m       _convert_nest_to_shapes(output_shape))\n\u001b[0;32m    152\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_load_options \u001b[38;5;241m=\u001b[39m load_options\n\u001b[1;32m--> 153\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_func \u001b[38;5;241m=\u001b[39m \u001b[43mload_module\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtags\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_load_options\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    154\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_has_training_argument \u001b[38;5;241m=\u001b[39m func_has_training_argument(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_func)\n\u001b[0;32m    155\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_is_hub_module_v1 \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_func, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_is_hub_module_v1\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mFalse\u001b[39;00m)\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\tf-proper\\lib\\site-packages\\tensorflow_hub\\keras_layer.py:449\u001b[0m, in \u001b[0;36mload_module\u001b[1;34m(handle, tags, load_options)\u001b[0m\n\u001b[0;32m    447\u001b[0m   \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m:  \u001b[38;5;66;03m# Expected before TF2.4.\u001b[39;00m\n\u001b[0;32m    448\u001b[0m     set_load_options \u001b[38;5;241m=\u001b[39m load_options\n\u001b[1;32m--> 449\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmodule_v2\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtags\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtags\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mset_load_options\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\tf-proper\\lib\\site-packages\\tensorflow_hub\\module_v2.py:92\u001b[0m, in \u001b[0;36mload\u001b[1;34m(handle, tags, options)\u001b[0m\n\u001b[0;32m     90\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(handle, \u001b[38;5;28mstr\u001b[39m):\n\u001b[0;32m     91\u001b[0m   \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpected a string, got \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m handle)\n\u001b[1;32m---> 92\u001b[0m module_path \u001b[38;5;241m=\u001b[39m \u001b[43mresolve\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     93\u001b[0m is_hub_module_v1 \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mio\u001b[38;5;241m.\u001b[39mgfile\u001b[38;5;241m.\u001b[39mexists(\n\u001b[0;32m     94\u001b[0m     native_module\u001b[38;5;241m.\u001b[39mget_module_proto_path(module_path))\n\u001b[0;32m     95\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m tags \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m is_hub_module_v1:\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\tf-proper\\lib\\site-packages\\tensorflow_hub\\module_v2.py:47\u001b[0m, in \u001b[0;36mresolve\u001b[1;34m(handle)\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mresolve\u001b[39m(handle):\n\u001b[0;32m     24\u001b[0m   \u001b[38;5;124;03m\"\"\"Resolves a module handle into a path.\u001b[39;00m\n\u001b[0;32m     25\u001b[0m \n\u001b[0;32m     26\u001b[0m \u001b[38;5;124;03m  This function works both for plain TF2 SavedModels and the legacy TF1 Hub\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     45\u001b[0m \u001b[38;5;124;03m    A string representing the Module path.\u001b[39;00m\n\u001b[0;32m     46\u001b[0m \u001b[38;5;124;03m  \"\"\"\u001b[39;00m\n\u001b[1;32m---> 47\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mregistry\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresolver\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\tf-proper\\lib\\site-packages\\tensorflow_hub\\registry.py:51\u001b[0m, in \u001b[0;36mMultiImplRegister.__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m     49\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m impl \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mreversed\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_impls):\n\u001b[0;32m     50\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m impl\u001b[38;5;241m.\u001b[39mis_supported(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m---> 51\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m     52\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m     53\u001b[0m     fails\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;28mtype\u001b[39m(impl)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m)\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\tf-proper\\lib\\site-packages\\tensorflow_hub\\compressed_module_resolver.py:67\u001b[0m, in \u001b[0;36mHttpCompressedFileResolver.__call__\u001b[1;34m(self, handle)\u001b[0m\n\u001b[0;32m     63\u001b[0m   response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_urlopen(request)\n\u001b[0;32m     64\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m resolver\u001b[38;5;241m.\u001b[39mDownloadManager(handle)\u001b[38;5;241m.\u001b[39mdownload_and_uncompress(\n\u001b[0;32m     65\u001b[0m       response, tmp_dir)\n\u001b[1;32m---> 67\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mresolver\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43matomic_download\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdownload\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodule_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     68\u001b[0m \u001b[43m                                \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_lock_file_timeout_sec\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\tf-proper\\lib\\site-packages\\tensorflow_hub\\resolver.py:418\u001b[0m, in \u001b[0;36matomic_download\u001b[1;34m(handle, download_fn, module_dir, lock_file_timeout_sec)\u001b[0m\n\u001b[0;32m    416\u001b[0m logging\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDownloading TF-Hub Module \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m, handle)\n\u001b[0;32m    417\u001b[0m tf\u001b[38;5;241m.\u001b[39mcompat\u001b[38;5;241m.\u001b[39mv1\u001b[38;5;241m.\u001b[39mgfile\u001b[38;5;241m.\u001b[39mMakeDirs(tmp_dir)\n\u001b[1;32m--> 418\u001b[0m \u001b[43mdownload_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtmp_dir\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    419\u001b[0m \u001b[38;5;66;03m# Write module descriptor to capture information about which module was\u001b[39;00m\n\u001b[0;32m    420\u001b[0m \u001b[38;5;66;03m# downloaded by whom and when. The file stored at the same level as a\u001b[39;00m\n\u001b[0;32m    421\u001b[0m \u001b[38;5;66;03m# directory in order to keep the content of the 'model_dir' exactly as it\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    426\u001b[0m \u001b[38;5;66;03m# module caching protocol and no code in the TF-Hub library reads its\u001b[39;00m\n\u001b[0;32m    427\u001b[0m \u001b[38;5;66;03m# content.\u001b[39;00m\n\u001b[0;32m    428\u001b[0m _write_module_descriptor_file(handle, module_dir)\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\tf-proper\\lib\\site-packages\\tensorflow_hub\\compressed_module_resolver.py:64\u001b[0m, in \u001b[0;36mHttpCompressedFileResolver.__call__.<locals>.download\u001b[1;34m(handle, tmp_dir)\u001b[0m\n\u001b[0;32m     61\u001b[0m request \u001b[38;5;241m=\u001b[39m urllib\u001b[38;5;241m.\u001b[39mrequest\u001b[38;5;241m.\u001b[39mRequest(\n\u001b[0;32m     62\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_append_compressed_format_query(handle))\n\u001b[0;32m     63\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_urlopen(request)\n\u001b[1;32m---> 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mresolver\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mDownloadManager\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdownload_and_uncompress\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m     65\u001b[0m \u001b[43m    \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtmp_dir\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\tf-proper\\lib\\site-packages\\tensorflow_hub\\resolver.py:189\u001b[0m, in \u001b[0;36mDownloadManager.download_and_uncompress\u001b[1;34m(self, fileobj, dst_path)\u001b[0m\n\u001b[0;32m    179\u001b[0m \u001b[38;5;124;03m\"\"\"Streams the content for the 'fileobj' and stores the result in dst_path.\u001b[39;00m\n\u001b[0;32m    180\u001b[0m \n\u001b[0;32m    181\u001b[0m \u001b[38;5;124;03mArgs:\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    186\u001b[0m \u001b[38;5;124;03m  ValueError: Unknown object encountered inside the TAR file.\u001b[39;00m\n\u001b[0;32m    187\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    188\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 189\u001b[0m   \u001b[43mfile_utils\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mextract_tarfile_to_destination\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    190\u001b[0m \u001b[43m      \u001b[49m\u001b[43mfileobj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdst_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlog_function\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_log_progress\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    191\u001b[0m   total_size_str \u001b[38;5;241m=\u001b[39m tf_utils\u001b[38;5;241m.\u001b[39mbytes_to_readable_str(\n\u001b[0;32m    192\u001b[0m       \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_total_bytes_downloaded, \u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m    193\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_print_download_progress_msg(\n\u001b[0;32m    194\u001b[0m       \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDownloaded \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m, Total size: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_url, total_size_str),\n\u001b[0;32m    195\u001b[0m       flush\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\tf-proper\\lib\\site-packages\\tensorflow_hub\\file_utils.py:52\u001b[0m, in \u001b[0;36mextract_tarfile_to_destination\u001b[1;34m(fileobj, dst_path, log_function)\u001b[0m\n\u001b[0;32m     49\u001b[0m abs_target_path \u001b[38;5;241m=\u001b[39m merge_relative_path(dst_path, tarinfo\u001b[38;5;241m.\u001b[39mname)\n\u001b[0;32m     51\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m tarinfo\u001b[38;5;241m.\u001b[39misfile():\n\u001b[1;32m---> 52\u001b[0m   \u001b[43mextract_file\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtgz\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtarinfo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mabs_target_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlog_function\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlog_function\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     53\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m tarinfo\u001b[38;5;241m.\u001b[39misdir():\n\u001b[0;32m     54\u001b[0m   tf\u001b[38;5;241m.\u001b[39mcompat\u001b[38;5;241m.\u001b[39mv1\u001b[38;5;241m.\u001b[39mgfile\u001b[38;5;241m.\u001b[39mMakeDirs(abs_target_path)\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\tf-proper\\lib\\site-packages\\tensorflow_hub\\file_utils.py:38\u001b[0m, in \u001b[0;36mextract_file\u001b[1;34m(tgz, tarinfo, dst_path, buffer_size, log_function)\u001b[0m\n\u001b[0;32m     36\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m buf:\n\u001b[0;32m     37\u001b[0m   \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[1;32m---> 38\u001b[0m \u001b[43mdst\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     39\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m log_function \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m     40\u001b[0m   log_function(\u001b[38;5;28mlen\u001b[39m(buf))\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python39\\site-packages\\tensorflow\\python\\lib\\io\\file_io.py:103\u001b[0m, in \u001b[0;36mFileIO.write\u001b[1;34m(self, file_content)\u001b[0m\n\u001b[0;32m    101\u001b[0m \u001b[38;5;124;03m\"\"\"Writes file_content to the file. Appends to the end of the file.\"\"\"\u001b[39;00m\n\u001b[0;32m    102\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_prewrite_check()\n\u001b[1;32m--> 103\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_writable_file\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mappend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcompat\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mas_bytes\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile_content\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[1;31mUnknownError\u001b[0m: Failed to WriteFile: C:\\Users\\Gurpr\\AppData\\Local\\Temp\\tfhub_modules\\063d866c06683311b44b4992fd46003be952409c.87dc21ef12df4d4997e0f0a03b89577a.tmp\\variables\\variables.data-00000-of-00001 : There is not enough space on the disk.\r\n; operation in progress"
     ]
    }
   ],
   "source": [
    "import tensorflow_hub as hub\n",
    "\n",
    "uni_sentence_encoder = hub.KerasLayer('https://tfhub.dev/google/universal-sentence-encoder/4',\n",
    "                                     name='uni_sentence_encoder')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a392a442",
   "metadata": {},
   "outputs": [],
   "source": [
    "random_train_sentence = random.choice(train_sentences)\n",
    "print(f\"Random sentence:\\n {random_train_sentence}\")\n",
    "use_embedded_sentence = uni_sentence_encoder([random_train_sentence])\n",
    "print(f\"Sentence after embedding:\\n {use_embedded_sentence[0][:30]}\")\n",
    "print(f\"Length of sentence embedding: {len(use_embedded_sentence[0])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "553100fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = layers.Input(shape=[], dtype=tf.string)\n",
    "pretrained_embedding = uni_sentence_encoder(inputs)\n",
    "x = layers.Dense(128, activation='relu')(pretrained_embedding)\n",
    "outputs = layers.Dense(num_classes, activation='softmax')(x)\n",
    "model_2 = tf.keras.Model(inputs, outputs, name='model_2_USE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3710e33e",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_2.compile(\n",
    "    optimizer=tf.keras.optimizers.Adam(),\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['accuracy']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ab42733",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_2.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "838765ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_2_history = model_2.fit(\n",
    "    train_dataset,\n",
    "    steps_per_epoch=int(0.1*len(train_dataset)),\n",
    "    validation_data = val_dataset,\n",
    "    epochs=5,\n",
    "    validation_steps=int(0.1 * len(val_dataset))\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "caa73b96",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_2.evaluate(val_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61621735",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_2_pred_probs = model_2.predict(val_dataset)\n",
    "model_2_pred_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e4c6c28",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_2_preds = tf.argmax(model_2_pred_probs, axis=1)\n",
    "model_2_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a739302b",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_2_results = calculate_results(val_labels_encoded, model_2_preds)\n",
    "model_2_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c768c551",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Model-3\n",
    "\n",
    "train_sentences[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecdf3ac9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_chars(text):\n",
    "    return \" \".join(list(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77018941",
   "metadata": {},
   "outputs": [],
   "source": [
    "split_chars(random_train_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d6876d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_chars = [split_chars(sentence) for sentence in train_sentences]\n",
    "val_chars = [split_chars(sentence) for sentence in val_sentences]\n",
    "test_chars = [split_chars(sentence) for sentence in test_sentences]\n",
    "\n",
    "print(train_chars[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "953172ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "chars_lens = [len(sentence) for sentence in train_sentences]\n",
    "mean_char_len = np.mean(chars_lens)\n",
    "mean_char_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1cd8cda",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(chars_lens, bins=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ee9ca11",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_seq_char_len = int(np.percentile(chars_lens, 95))\n",
    "output_seq_char_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91580f3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "alphabet = string.ascii_lowercase + string.digits + string.punctuation\n",
    "alphabet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8fe060ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_CHAR_TOKENS = len(alphabet)\n",
    "char_vectorizer = tf.keras.layers.TextVectorization(\n",
    "    max_tokens=NUM_CHAR_TOKENS,\n",
    "    output_sequence_length=output_seq_char_len,\n",
    "    name='char_vectorizer'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ec03f31",
   "metadata": {},
   "outputs": [],
   "source": [
    "char_vectorizer.adapt(train_chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e17ea18",
   "metadata": {},
   "outputs": [],
   "source": [
    "char_vocab = char_vectorizer.get_vocabulary()\n",
    "print(f\"Number of different characters in character vocab: {len(char_vocab)}\")\n",
    "print(f\"5 most common characters: {char_vocab[:5]}\")\n",
    "print(f\"5 least common characters: {char_vocab[-5:]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b97fce85",
   "metadata": {},
   "outputs": [],
   "source": [
    "random_train_chars = random.choice(train_chars)\n",
    "print(f\"Charified text:\\n {random_train_chars}\")\n",
    "print(f\"Length of random_train_chars: {len(random_train_chars.split())}\")\n",
    "vectorized_chars = char_vectorizer([random_train_chars])\n",
    "print(f\"\\Vectorized chars:\\n {vectorized_chars}\")\n",
    "print(f\"Vectorized chars shape: {vectorized_chars.shape}\")\n",
    "print(f\"\\nLength of vectorized chars: {len(vectorized_chars[0])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "072a01dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "char_embed = layers.Embedding(\n",
    "    input_dim = len(char_vocab),\n",
    "    output_dim=25,\n",
    "    mask_zero=True,\n",
    "    name='char_embed'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68188d67",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"Charified text: {random_train_chars}\\n\")\n",
    "char_embed_example = char_embed(char_vectorizer([random_train_chars]))\n",
    "print(f\"Embedded chars:\\n {char_embed_example}\")\n",
    "print(f\"Character embedding shape: {char_embed_example.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aff3058c",
   "metadata": {},
   "outputs": [],
   "source": [
    "val_char_dataset = tf.data.Dataset.from_tensor_slices((val_chars, val_labels_one_hot)).batch(32).prefetch(tf.data.AUTOTUNE)\n",
    "train_char_dataset = tf.data.Dataset.from_tensor_slices((train_chars, train_labels_one_hot)).batch(32).prefetch(tf.data.AUTOTUNE)\n",
    "test_char_dataset = tf.data.Dataset.from_tensor_slices((test_chars, test_labels_one_hot)).batch(32).prefetch(tf.data.AUTOTUNE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "393738cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = layers.Input(shape=(1,), dtype=tf.string)\n",
    "x = char_vectorizer(inputs)\n",
    "x = char_embed(x)\n",
    "x = layers.Conv1D(64,5,activation='relu')(x)\n",
    "x = layers.GlobalMaxPooling1D()(x)\n",
    "outputs = layers.Dense(num_classes, activation='softmax')(x)\n",
    "model_3 = tf.keras.Model(inputs, outputs, name='model_3_Conv1D_Char')\n",
    "\n",
    "model_3.compile(\n",
    "    optimizer=tf.keras.optimizers.Adam(),\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['accuracy']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aee93e33",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_3.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b18b75f",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_3_history = model_3.fit(\n",
    "    train_dataset,\n",
    "    steps_per_epoch=int(0.1*len(train_char_dataset)),\n",
    "    epochs=5,\n",
    "    validation_data=val_dataset,\n",
    "    validation_steps=int(0.1*len(val_char_dataset))\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a99d4503",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_3.evaluate(val_char_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08a22a54",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_3_pred_probs = model_3.predict(val_char_dataset)\n",
    "model_3_pred_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98eb0c3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_3_preds = tf.argmax(model_3_pred_probs, axis=1)\n",
    "model_3_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12e0b0b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_3_results = calculate_results(val_labels_encoded, model_3_preds)\n",
    "model_3_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e333c58",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Model-4\n",
    "\n",
    "#Create a token-level embedding model\n",
    "#Create a character level model\n",
    "#Combine both models with a concatenate\n",
    "#Build a series of output layers on top of the combo\n",
    "#Construct a model which takes token and character level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "509cc64f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Token model\n",
    "token_inputs = layers.Input(shape=[], dtype=tf.string, name='token_input')\n",
    "token_embeddings = uni_sentence_encoder(token_inputs)\n",
    "token_outputs = layers.Dense(128, activation='relu')(token_embeddings)\n",
    "token_model = tf.keras.Model(token_inputs, token_outputs)\n",
    "\n",
    "#Char model\n",
    "char_inputs = layers.Input(shape=(1,), dtype=tf.string)\n",
    "char_vectors = char_vectorizer(char_inputs)\n",
    "char_embeddings = char_embed(char_vectors)\n",
    "char_bi_lstm = layers.Bidirectional(layers.LSTM(24))(char_embeddings)\n",
    "char_model = tf.keras.Model(char_inputs, char_bi_lstm)\n",
    "\n",
    "#Concatenate\n",
    "token_char_concat = tf.keras.layers.Concatenate(name='token_char_hybrid')([token_model.output, char_model.output])\n",
    "\n",
    "combined_dropout = layers.Dropout(0.5)(token_char_concat)\n",
    "combined_dense = layers.Dense(128, activation='relu')(combined_dropout)\n",
    "final_dropout = layers.Dropout(0.5)(combined_dense)\n",
    "output_layer = layers.Dense(num_classes, activation='softmax')(final_dropout)\n",
    "\n",
    "model_4 = tf.keras.Model(inputs=[token_model.input, char_model.input], outputs=output_layer, name='model_4_token_and_char_embeddings')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6dccd824",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_4.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4eb004ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_4.compile(\n",
    "    loss='categorical_crossentropy',\n",
    "    optimizer=tf.keras.optimizers.Adam(),\n",
    "    metrics=['accuracy']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09a2d854",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_char_token_data = tf.data.Dataset.from_tensor_slices((train_sentences, train_chars))\n",
    "train_char_token_labels = tf.data.Dataset.from_tensor_slices(train_labels_one_hot)\n",
    "train_char_token_dataset = tf.data.Dataset.zip((train_char_token_data, train_char_token_labels))\n",
    "\n",
    "train_char_token_dataset = train_char_token_dataset.batch(32).prefetch(tf.data.AUTOTUNE)\n",
    "\n",
    "val_char_token_data = tf.data.Dataset.from_tensor_slices((val_sentences, val_chars))\n",
    "val_char_token_labels = tf.data.Dataset.from_tensor_slices(val_labels_one_hot)\n",
    "val_char_token_dataset = tf.data.Dataset.zip((val_char_token_data, val_char_token_labels))\n",
    "\n",
    "val_char_token_dataset = val_char_token_dataset.batch(32).prefetch(tf.data.AUTOTUNE)\n",
    "\n",
    "\n",
    "test_char_token_data = tf.data.Dataset.from_tensor_slices((test_sentences, test_chars))\n",
    "test_char_token_labels = tf.data.Dataset.from_tensor_slices(test_labels_one_hot)\n",
    "test_char_token_dataset = tf.data.Dataset.zip((test_char_token_data, test_char_token_labels))\n",
    "\n",
    "test_char_token_dataset = test_char_token_dataset.batch(32).prefetch(tf.data.AUTOTUNE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "362c0d3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_char_token_dataset, val_char_token_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "039fc958",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_4_history = model_4.fit(\n",
    "    train_char_token_dataset,\n",
    "    steps_per_epoch=int(0.1*len(train_char_token_dataset)),\n",
    "    epochs=5,\n",
    "    validation_data = val_char_token_dataset,\n",
    "    validation_steps=int(0.1*len(val_char_token_dataset))\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6dd0b5b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_4.evaluate(val_char_token_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d06124f",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_4_pred_probs = model_4.predict(val_char_token_dataset)\n",
    "model_4_pred_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d53595d",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_4_preds = tf.argmax(model_4_pred_probs, axis=1)\n",
    "model_4_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "992fe1d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_4_results = calculate_results(val_labels_encoded, model_4_preds)\n",
    "model_4_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "582846a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "baseline_results, model_1_results, model_2_results, model_3_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e83e97f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Model-5\n",
    "\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dab62d17",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df['line_number'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f653398",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.line_number.plot.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72fbd894",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_line_numbers_one_hot = tf.one_hot(train_df['line_number'].to_numpy(), depth=15)\n",
    "val_line_numbers_one_hot = tf.one_hot(val_df['line_number'].to_numpy(), depth=15)\n",
    "test_line_numbers_one_hot = tf.one_hot(test_df['line_number'].to_numpy(), depth=15)\n",
    "\n",
    "train_line_numbers_one_hot[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f9e5ce9",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df['total_lines'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15d36074",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.total_lines.plot.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f570775e",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.percentile(train_df.total_lines, 98)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ce204a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_total_lines_one_hot = tf.one_hot(train_df['total_lines'].to_numpy(), depth=20)\n",
    "val_total_lines_one_hot = tf.one_hot(val_df['total_lines'].to_numpy(), depth=20)\n",
    "test_total_lines_one_hot = tf.one_hot(test_df['total_lines'].to_numpy(), depth=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e466289c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create a token-level model\n",
    "#Create a character-level model\n",
    "#Create a model for the line number feature\n",
    "#Create a model for the total lines feature\n",
    "#Combine the outputs of the first two models using concatenate\n",
    "#Combine the features and the combined models using concatenate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d317fa2",
   "metadata": {},
   "outputs": [],
   "source": [
    "token_inputs = layers.Input(shape=[], dtype=tf.string, name='token_inputs')\n",
    "token_embeddings = uni_sentence_encoder(token_inputs)\n",
    "token_outputs = layers.Dense(128, activation='relu')(token_embeddings)\n",
    "token_model = tf.keras.Model(token_inputs, token_outputs)\n",
    "\n",
    "char_inputs = layers.Input(shape=(1,), dtype=tf.string, name='char_inputs')\n",
    "char_vectors = char_vectorizer(char_inputs)\n",
    "char_embeddings = char_embed(char_vectors)\n",
    "char_bi_lstm = layers.Bidirectional(layers.LSTM(24))(char_embeddings)\n",
    "char_model = tf.keras.Model(char_inputs, char_bi_lstm)\n",
    "\n",
    "line_number_inputs = layers.Input(shape=(15,), dtype=tf.float32, name='line_number_input')\n",
    "x = layers.Dense(32, activation='relu')(line_number_inputs)\n",
    "line_number_model = tf.keras.Model(line_number_inputs, x)\n",
    "\n",
    "total_lines_inputs = layers.Input(shape=(20), dtype=tf.float32, name='total_lines_input')\n",
    "y = layers.Dense(32, activation='relu')(total_lines_inputs)\n",
    "total_line_model = tf.keras.Model(total_lines_inputs, y)\n",
    "\n",
    "combined_embeddings = layers.Concatenate(name='char_token_hybrid_embedding')([token_model.output, char_model.output])\n",
    "\n",
    "z = layers.Dense(256, activation='relu')(combined_embeddings)\n",
    "z = layers.Dropout(0.5)(z)\n",
    "\n",
    "tribrid_embeddings = layers.Concatenate(name='char_token_positional_embedding')([line_number_model.output, total_line_model.output, z])\n",
    "\n",
    "output_layer = layers.Dense(num_classes, activation='softmax', name='output_layer')(tribrid_embeddings)\n",
    "\n",
    "model_5 = tf.keras.Model([line_number_model.input,\n",
    "                         total_line_model.input,\n",
    "                         token_model.input,\n",
    "                         char_model.input], outputs=output_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebdcfd50",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_5.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99cfc49b",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_5.compile(\n",
    "    loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.2),\n",
    "    optimizer=tf.keras.optimizers.Adam(),\n",
    "    metrics=['accuracy']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24213078",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create training and validation datasets using all four sets of data\n",
    "\n",
    "train_char_token_pos_data = tf.data.Dataset.from_tensor_slices((train_line_numbers_one_hot,\n",
    "                                                               train_total_lines_one_hot,\n",
    "                                                               train_sentences,\n",
    "                                                               train_chars))\n",
    "\n",
    "train_char_token_pos_labels = tf.data.Dataset.from_tensor_slices(train_labels_one_hot)\n",
    "train_char_token_pos_dataset = tf.data.Dataset.zip((train_char_token_pos_data, train_char_token_pos_labels))\n",
    "train_char_token_pos_dataset = train_char_token_pos_dataset.batch(32).prefetch(tf.data.AUTOTUNE)\n",
    "\n",
    "val_char_token_pos_data = tf.data.Dataset.from_tensor_slices((val_line_numbers_one_hot,\n",
    "                                                               val_total_lines_one_hot,\n",
    "                                                               val_sentences,\n",
    "                                                               val_chars))\n",
    "\n",
    "val_char_token_pos_labels = tf.data.Dataset.from_tensor_slices(val_labels_one_hot)\n",
    "val_char_token_pos_dataset = tf.data.Dataset.zip((val_char_token_pos_data, val_char_token_pos_labels))\n",
    "val_char_token_pos_dataset = val_char_token_pos_dataset.batch(32).prefetch(tf.data.AUTOTUNE)\n",
    "\n",
    "test_char_token_pos_data = tf.data.Dataset.from_tensor_slices((test_line_numbers_one_hot,\n",
    "                                                               test_total_lines_one_hot,\n",
    "                                                               test_sentences,\n",
    "                                                               test_chars))\n",
    "\n",
    "test_char_token_pos_labels = tf.data.Dataset.from_tensor_slices(test_labels_one_hot)\n",
    "test_char_token_pos_dataset = tf.data.Dataset.zip((test_char_token_pos_data, test_char_token_pos_labels))\n",
    "test_char_token_pos_dataset = test_char_token_pos_dataset.batch(32).prefetch(tf.data.AUTOTUNE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c81e020",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_char_token_pos_dataset, val_char_token_pos_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c328394c",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_5_history = model_5.fit(\n",
    "    train_char_token_pos_dataset,\n",
    "    epochs=5,\n",
    "    steps_per_epoch=int(0.1*len(train_char_token_pos_dataset)),\n",
    "    validation_data=val_char_token_pos_dataset,\n",
    "    validation_steps = int(0.1*len(val_char_token_pos_dataset)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d53ff6be",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_5.evaluate(val_char_token_pos_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da162917",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_5_pred_probs = model_5.predict(val_char_token_pos_dataset)\n",
    "model_5_pred_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e81dfa7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_5_preds = tf.argmax(model_5_pred_probs, axis=1)\n",
    "model_5_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d279952f",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_5_results = calculate_results(val_labels_encoded, model_5_preds)\n",
    "model_5_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0cabd21",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42292ddc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}