03-train/03-oob-without-finetuning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use `original` BERT to take `fill mask` test without fine-tuning it on our COVID articles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prerequisites"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture \n",
    "\n",
    "!pip install transformers==4.17.0\n",
    "!pip install pandas==1.1.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Imports "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import BertTokenizerFast\n",
    "from transformers import BertForMaskedLM\n",
    "from transformers import BertConfig\n",
    "from transformers import pipeline\n",
    "import transformers \n",
    "import pandas as pd\n",
    "import logging"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Setup logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger = logging.getLogger('sagemaker')\n",
    "logger.setLevel(logging.DEBUG)\n",
    "logger.addHandler(logging.StreamHandler())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Log versions of dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Using transformers: 4.17.0]\n",
      "[Using pandas: 1.1.5]\n"
     ]
    }
   ],
   "source": [
    "logger.info(f'[Using transformers: {transformers.__version__}]')\n",
    "logger.info(f'[Using pandas: {pd.__version__}]')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Essentials"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Re-create BERT MLM "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Total number of parameters = 109514298\n"
     ]
    }
   ],
   "source": [
    "oob_model = BertForMaskedLM.from_pretrained('bert-base-uncased')\n",
    "logger.info(f'Total number of parameters = {oob_model.num_parameters()}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Re-create default BERT tokenizer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PreTrainedTokenizerFast(name_or_path='bert-base-uncased', vocab_size=30522, model_max_len=512, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oob_tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')\n",
    "oob_tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Verify tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'unk_token': '[UNK]',\n",
       " 'sep_token': '[SEP]',\n",
       " 'pad_token': '[PAD]',\n",
       " 'cls_token': '[CLS]',\n",
       " 'mask_token': '[MASK]'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oob_tokenizer.special_tokens_map"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create HuggingFace Pipeline for `fill mask` task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "fill_mask = pipeline('fill-mask', model=oob_model, tokenizer=oob_tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test original BERT MLM for `fill mask` task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ground Truth    : A number of firms have been reassessing spending plans in light of the covid-19 outbreak and reduced oil price.\n",
      "Masked sentence : A number of firms have been reassessing spending plans in light of the covid-19 [MASK] and reduced oil price.\n",
      "[2022-09-21 17:15:52.850 pytorch-1-8-gpu-py3-ml-g4dn-xlarge-60bd0d07a83be181dcf7335baae2:11847 INFO utils.py:27] RULE_JOB_STOP_SIGNAL_FILENAME: None\n",
      "[2022-09-21 17:15:52.946 pytorch-1-8-gpu-py3-ml-g4dn-xlarge-60bd0d07a83be181dcf7335baae2:11847 INFO profiler_config_parser.py:102] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\n",
      "Rank: 1 | 11.37 % | ['crisis']\n",
      "Rank: 2 | 8.95 % | ['crash']\n",
      "Rank: 3 | 3.26 % | ['program']\n",
      "Rank: 4 | 2.08 % | ['earthquake']\n",
      "Rank: 5 | 1.98 % | ['agreement']\n",
      "Rank: 6 | 1.91 % | ['disaster']\n",
      "Rank: 7 | 1.75 % | ['accident']\n",
      "Rank: 8 | 1.62 % | ['treaty']\n",
      "Rank: 9 | 1.31 % | ['ban']\n",
      "Rank: 10 | 1.23 % | ['issue']\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Ground Truth    : Globally, airlines are closing down and the covid-19 coronavirus has accelerated some of these closures.\n",
      "Masked sentence : Globally, airlines are closing down and the covid-19 [MASK] has accelerated some of these closures.\n",
      "Rank: 1 | 22.87 % | ['program']\n",
      "Rank: 2 | 11.54 % | ['system']\n",
      "Rank: 3 | 10.48 % | ['project']\n",
      "Rank: 4 | 5.53 % | ['programme']\n",
      "Rank: 5 | 1.20 % | ['upgrade']\n",
      "Rank: 6 | 1.19 % | ['initiative']\n",
      "Rank: 7 | 1.06 % | ['technology']\n",
      "Rank: 8 | 1.06 % | ['##b']\n",
      "Rank: 9 | 1.04 % | ['network']\n",
      "Rank: 10 | 1.03 % | ['satellite']\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Ground Truth    : The first human challenge trial for covid has given some helpful insights into how infection progresses.\n",
      "Masked sentence : The first human challenge trial for covid has given some helpful insights into how [MASK] progresses.\n",
      "Rank: 1 | 14.47 % | ['it']\n",
      "Rank: 2 | 8.71 % | ['development']\n",
      "Rank: 3 | 8.66 % | ['disease']\n",
      "Rank: 4 | 4.48 % | ['research']\n",
      "Rank: 5 | 4.00 % | ['replication']\n",
      "Rank: 6 | 3.92 % | ['treatment']\n",
      "Rank: 7 | 3.43 % | ['infection']\n",
      "Rank: 8 | 3.40 % | ['cancer']\n",
      "Rank: 9 | 2.49 % | ['hiv']\n",
      "Rank: 10 | 2.09 % | ['evolution']\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Ground Truth    : The question of whether masks will reduce transmission of covid-19 in the general public is contested.\n",
      "Masked sentence : The question of whether [MASK] will reduce transmission of covid-19 in the general public is contested.\n",
      "Rank: 1 | 46.84 % | ['this']\n",
      "Rank: 2 | 32.49 % | ['it']\n",
      "Rank: 3 | 1.57 % | ['treatment']\n",
      "Rank: 4 | 1.02 % | ['vaccines']\n",
      "Rank: 5 | 0.77 % | ['they']\n",
      "Rank: 6 | 0.68 % | ['that']\n",
      "Rank: 7 | 0.58 % | ['screening']\n",
      "Rank: 8 | 0.41 % | ['smoking']\n",
      "Rank: 9 | 0.38 % | ['chemotherapy']\n",
      "Rank: 10 | 0.37 % | ['these']\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Ground Truth    : CDC estimates more than 9,200 health care workers have been infected with covid-19\n",
      "Masked sentence : [MASK] estimates more than 9,200 health care workers have been infected with covid-19\n",
      "Rank: 1 | 26.65 % | ['unicef']\n",
      "Rank: 2 | 14.44 % | ['it']\n",
      "Rank: 3 | 10.95 % | ['cdc']\n",
      "Rank: 4 | 5.59 % | ['research']\n",
      "Rank: 5 | 3.60 % | ['who']\n",
      "Rank: 6 | 2.05 % | ['he']\n",
      "Rank: 7 | 2.03 % | ['one']\n",
      "Rank: 8 | 1.09 % | ['researchers']\n",
      "Rank: 9 | 1.03 % | ['iran']\n",
      "Rank: 10 | 0.83 % | ['fda']\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Ground Truth    : A drug called remdesivir appears to actually work against the coronavirus that causes covid-19.\n",
      "Masked sentence : A drug called [MASK] appears to actually work against the coronavirus that causes covid-19.\n",
      "Rank: 1 | 2.30 % | ['ace']\n",
      "Rank: 2 | 0.64 % | ['insulin']\n",
      "Rank: 3 | 0.52 % | ['hope']\n",
      "Rank: 4 | 0.43 % | ['it']\n",
      "Rank: 5 | 0.41 % | ['eta']\n",
      "Rank: 6 | 0.40 % | ['omega']\n",
      "Rank: 7 | 0.31 % | ['adrenaline']\n",
      "Rank: 8 | 0.28 % | ['ami']\n",
      "Rank: 9 | 0.28 % | ['beta']\n",
      "Rank: 10 | 0.27 % | ['sam']\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Ground Truth    : Israel has data showing 64% effectiveness of full vaccination with Pfizer vaccine against infection with covid ( delta variant).\n",
      "Masked sentence : Israel has data showing 64% effectiveness of full vaccination with [MASK] vaccine against infection with covid ( delta variant).\n",
      "Rank: 1 | 37.84 % | ['a']\n",
      "Rank: 2 | 22.19 % | ['the']\n",
      "Rank: 3 | 5.61 % | ['this']\n",
      "Rank: 4 | 5.40 % | ['hiv']\n",
      "Rank: 5 | 3.12 % | ['influenza']\n",
      "Rank: 6 | 2.03 % | ['its']\n",
      "Rank: 7 | 1.55 % | ['flu']\n",
      "Rank: 8 | 0.75 % | ['oral']\n",
      "Rank: 9 | 0.58 % | ['smallpox']\n",
      "Rank: 10 | 0.51 % | ['full']\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Ground Truth    : A rising number of cases of the omicron variant in India have policymakers preparing for another potential covid-19 battle.\n",
      "Masked sentence : A rising number of cases of the [MASK] variant in India have policymakers preparing for another potential covid-19 battle.\n",
      "Rank: 1 | 6.03 % | ['same']\n",
      "Rank: 2 | 5.45 % | ['new']\n",
      "Rank: 3 | 1.97 % | ['latter']\n",
      "Rank: 4 | 1.54 % | ['russian']\n",
      "Rank: 5 | 1.05 % | ['venom']\n",
      "Rank: 6 | 1.04 % | ['proposed']\n",
      "Rank: 7 | 0.74 % | ['indian']\n",
      "Rank: 8 | 0.70 % | ['military']\n",
      "Rank: 9 | 0.69 % | ['lightning']\n",
      "Rank: 10 | 0.65 % | ['current']\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Ground Truth    : Africa suffers worst surge in Covid cases as delta variant spurs third wave of pandemic\n",
      "Masked sentence : Africa suffers worst surge in Covid cases as [MASK] variant spurs third wave of pandemic\n",
      "Rank: 1 | 28.34 % | ['this']\n",
      "Rank: 2 | 8.61 % | ['the']\n",
      "Rank: 3 | 4.95 % | ['its']\n",
      "Rank: 4 | 2.65 % | ['a']\n",
      "Rank: 5 | 2.51 % | ['african']\n",
      "Rank: 6 | 2.27 % | ['genetic']\n",
      "Rank: 7 | 2.05 % | ['virus']\n",
      "Rank: 8 | 2.00 % | ['one']\n",
      "Rank: 9 | 1.39 % | ['human']\n",
      "Rank: 10 | 0.87 % | ['their']\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Ground Truth    : The ongoing coronavirus pandemic and spread of the highly transmissible delta Covid-19 variant has rattled investor sentiment lately.\n",
      "Masked sentence : The ongoing coronavirus pandemic and spread of the highly transmissible [MASK] Covid-19 variant has rattled investor sentiment lately.\n",
      "Rank: 1 | 18.58 % | ['human']\n",
      "Rank: 2 | 8.99 % | ['vaccine']\n",
      "Rank: 3 | 7.97 % | [',']\n",
      "Rank: 4 | 6.82 % | ['viral']\n",
      "Rank: 5 | 5.42 % | ['virus']\n",
      "Rank: 6 | 4.11 % | ['hiv']\n",
      "Rank: 7 | 2.40 % | ['vector']\n",
      "Rank: 8 | 2.04 % | ['mutant']\n",
      "Rank: 9 | 1.78 % | ['resistant']\n",
      "Rank: 10 | 1.61 % | ['inactivated']\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Ground Truth    : WHO says omicron variant could change the course of the Covid pandemic\n",
      "Masked sentence : WHO says omicron [MASK] could change the course of the Covid pandemic\n",
      "Rank: 1 | 8.04 % | ['alone']\n",
      "Rank: 2 | 5.30 % | ['itself']\n",
      "Rank: 3 | 4.29 % | ['2']\n",
      "Rank: 4 | 3.55 % | ['radiation']\n",
      "Rank: 5 | 3.36 % | ['##is']\n",
      "Rank: 6 | 2.26 % | ['3']\n",
      "Rank: 7 | 1.85 % | ['##us']\n",
      "Rank: 8 | 1.66 % | ['ii']\n",
      "Rank: 9 | 1.35 % | ['1']\n",
      "Rank: 10 | 1.25 % | ['iii']\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Ground Truth    : The Delta coronavirus variant has made Covid-19 cases in children more common than during the onset of the pandemic.\n",
      "Masked sentence : The Delta [MASK] variant has made Covid-19 cases in children more common than during the onset of the pandemic.\n",
      "Rank: 1 | 10.74 % | ['##1']\n",
      "Rank: 2 | 9.60 % | ['-']\n",
      "Rank: 3 | 4.99 % | ['##2']\n",
      "Rank: 4 | 2.83 % | ['##3']\n",
      "Rank: 5 | 1.85 % | ['cell']\n",
      "Rank: 6 | 1.81 % | ['protein']\n",
      "Rank: 7 | 1.59 % | ['gene']\n",
      "Rank: 8 | 1.44 % | ['1']\n",
      "Rank: 9 | 1.43 % | ['3']\n",
      "Rank: 10 | 1.40 % | ['2']\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Ground Truth    : The BA.2 variant is forcing substantial shifts in how the U.S. is trying to control the pandemic.\n",
      "Masked sentence : The [MASK] variant is forcing substantial shifts in how the U.S. is trying to control the pandemic.\n",
      "Rank: 1 | 21.66 % | ['new']\n",
      "Rank: 2 | 9.79 % | ['second']\n",
      "Rank: 3 | 4.20 % | ['current']\n",
      "Rank: 4 | 3.60 % | ['third']\n",
      "Rank: 5 | 2.21 % | ['latest']\n",
      "Rank: 6 | 2.11 % | ['global']\n",
      "Rank: 7 | 1.61 % | ['first']\n",
      "Rank: 8 | 1.44 % | ['international']\n",
      "Rank: 9 | 1.39 % | ['latter']\n",
      "Rank: 10 | 1.38 % | ['fourth']\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Ground Truth    : In fact, the BA.2 variant now accounts for 72% of new cases in the U.S.\n",
      "Masked sentence : In fact, the [MASK] variant now accounts for 72% of new cases in the U.S.\n",
      "Rank: 1 | 5.61 % | ['new']\n",
      "Rank: 2 | 1.50 % | ['modified']\n",
      "Rank: 3 | 1.46 % | ['older']\n",
      "Rank: 4 | 1.22 % | ['american']\n",
      "Rank: 5 | 1.17 % | ['latter']\n",
      "Rank: 6 | 1.10 % | ['hybrid']\n",
      "Rank: 7 | 1.03 % | ['standard']\n",
      "Rank: 8 | 0.98 % | ['original']\n",
      "Rank: 9 | 0.87 % | ['newer']\n",
      "Rank: 10 | 0.83 % | ['genetic']\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Ground Truth    : Pfizer and Moderna are both testing new formulations of their vaccine tailored to the omicron variant.\n",
      "Masked sentence : Pfizer and [MASK] are both testing new formulations of their vaccine tailored to the omicron variant.\n",
      "Rank: 1 | 21.78 % | ['dupont']\n",
      "Rank: 2 | 12.06 % | ['bayer']\n",
      "Rank: 3 | 4.85 % | ['roche']\n",
      "Rank: 4 | 2.77 % | ['fda']\n",
      "Rank: 5 | 2.34 % | ['target']\n",
      "Rank: 6 | 2.32 % | ['sony']\n",
      "Rank: 7 | 2.23 % | ['ge']\n",
      "Rank: 8 | 2.19 % | ['bosch']\n",
      "Rank: 9 | 1.84 % | ['cdc']\n",
      "Rank: 10 | 1.64 % | ['toyota']\n",
      "----------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('./../data/eval_mlm.csv')\n",
    "\n",
    "for gt, masked_sentence in zip(df.ground_truth.tolist(), df.masked.tolist()):\n",
    "    print(f'Ground Truth    : {gt}')\n",
    "    print(f'Masked sentence : {masked_sentence}')\n",
    "    predictions = fill_mask(masked_sentence, top_k=10)\n",
    "    for i, prediction in enumerate(predictions):\n",
    "        print(f'Rank: {i+1} | {(prediction[\"score\"] * 100):.2f} % | {[prediction[\"token_str\"]]}')\n",
    "    print('-' * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "instance_type": "ml.g4dn.xlarge",
  "kernelspec": {
   "display_name": "Python 3 (PyTorch 1.8 Python 3.6 GPU Optimized)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/pytorch-1.8-gpu-py36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}