BERT_QWK.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BERT for MISL Score Prediction\n",
    "This code draws mostly from the following repository with code and hyperparameter changes for our specific dataset: https://github.com/ceshine/pytorch-pretrained-BERT/blob/master/notebooks/Sequence%20Regression%20Model.ipynb\n",
    "\n",
    "It is assumed that PyTorch (pytorch.org) is installed and a large GPU is highly recommended as BERT requires multiple GB of GPU memory. This paper used an NVIDIA GeForce GTX Titan X for all training. There is also an implementation of BERT in PyTorch that must be installed prior to running this notebook. The repository and installation instructions can be found here: https://github.com/huggingface/pytorch-pretrained-BERT.\n",
    "\n",
    "Comments have been made where changes must be made to replicate results on the users machine."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import logging\n",
    "\n",
    "logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',\n",
    "                    datefmt = '%m/%d/%Y %H:%M:%S',\n",
    "                    level = logging.INFO)\n",
    "logger = logging.getLogger(\"regressor\")\n",
    "\n",
    "FP16 = False\n",
    "#BATCH_SIZE = 16\n",
    "BATCH_SIZE = 8\n",
    "SEED = 42\n",
    "WARMUP_PROPORTION = 0.1\n",
    "PYTORCH_PRETRAINED_BERT_CACHE = \"/home/.../bert-output/\" # Cache directory\n",
    "LOSS_SCALE = 0. \n",
    "MAX_SEQ_LENGTH = 128\n",
    "\n",
    "DATA_PATH = \"/home/.../AutomatedNarrativeAnalysisMISLData.csv\" # Path to data set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from pytorch_pretrained_bert.modeling import BertPreTrainedModel, BertModel\n",
    "from pytorch_pretrained_bert.tokenization import BertTokenizer\n",
    "from pytorch_pretrained_bert.optimization import BertAdam, warmup_linear, SCHEDULES\n",
    "from fastprogress import master_bar, progress_bar\n",
    "from sklearn.model_selection import train_test_split\n",
    "import random\n",
    "import ml_metrics as metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "n_gpu = torch.cuda.device_count()\n",
    "logger.info(\"device: {} n_gpu: {}, 16-bits training: {}\".format(\n",
    "    device, n_gpu, FP16))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "if n_gpu > 0:\n",
    "    torch.cuda.manual_seed_all(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BertForSequenceRegression(BertPreTrainedModel):\n",
    "    def __init__(self, config):\n",
    "        super(BertForSequenceRegression, self).__init__(config)\n",
    "        self.bert = BertModel(config)\n",
    "        self.dropout = nn.Dropout(config.hidden_dropout_prob)\n",
    "        self.regressor = nn.Linear(config.hidden_size, 1)\n",
    "        self.apply(self.init_bert_weights)\n",
    "        self.loss_fct = torch.nn.MSELoss()\n",
    "\n",
    "    def forward(self, input_ids, token_type_ids=None, attention_mask=None, targets=None):\n",
    "        _, pooled_output = self.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False)\n",
    "        pooled_output = self.dropout(pooled_output)\n",
    "        outputs = self.regressor(pooled_output).clamp(0, 3)\n",
    "        if targets is not None:\n",
    "            loss = self.loss_fct(outputs.view(-1), targets.view(-1))\n",
    "            return loss\n",
    "        else:\n",
    "            return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class InputExample(object):\n",
    "    \"\"\"A single training/test example for simple sequence classification.\"\"\"\n",
    "\n",
    "    def __init__(self, guid, text, target=None):\n",
    "        self.guid = guid\n",
    "        self.text = text\n",
    "        self.target = target\n",
    "\n",
    "\n",
    "class InputFeatures(object):\n",
    "    \"\"\"A single set of features of data.\"\"\"\n",
    "\n",
    "    def __init__(self, input_ids, input_mask, segment_ids, target):\n",
    "        self.input_ids = input_ids\n",
    "        self.input_mask = input_mask\n",
    "        self.segment_ids = segment_ids\n",
    "        self.target = target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MISLMacroProcessor:\n",
    "    def __init__(self, macro_score):\n",
    "        \n",
    "        df = pd.read_csv(DATA_PATH)\n",
    "        texts = df[\"vecOfNarratives\"].values\n",
    "        scores = df[macro_score].values\n",
    "        self.x_train, self.x_valid, self.y_train, self.y_valid = train_test_split(texts, scores, test_size=0.2)\n",
    "        \n",
    "    def get_train_examples(self):\n",
    "        return self._create_examples(self.x_train, self.y_train)\n",
    "\n",
    "    def get_dev_examples(self):\n",
    "        return self._create_examples(self.x_valid, self.y_valid)\n",
    "\n",
    "    #def get_test_examples(self):\n",
    "    #    return self._create_examples(self.x_test, self.y_test)\n",
    "    \n",
    "    def _create_examples(self, x, y):\n",
    "        \"\"\"Creates examples for the training and dev sets.\"\"\"\n",
    "        examples = []\n",
    "        for (i, (texts, target)) in enumerate(zip(x, y)):\n",
    "            examples.append(\n",
    "                InputExample(guid=i, text=texts, target=target))\n",
    "        return examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_examples_to_features(examples, max_seq_length, tokenizer):\n",
    "    \"\"\"Loads a data file into a list of `InputBatch`s.\"\"\"\n",
    "    \n",
    "    features = []\n",
    "    for (ex_index, example) in enumerate(examples):\n",
    "        tokens = tokenizer.tokenize(example.text)\n",
    "        \n",
    "        if len(tokens) > max_seq_length - 2:\n",
    "            tokens = tokens[:(max_seq_length - 2)]\n",
    "\n",
    "        # The convention in BERT is:\n",
    "        # (a) For sequence pairs:\n",
    "        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]\n",
    "        #  type_ids: 0   0  0    0    0     0       0 0    1  1  1  1   1 1\n",
    "        # (b) For single sequences:\n",
    "        #  tokens:   [CLS] the dog is hairy . [SEP]\n",
    "        #  type_ids: 0   0   0   0  0     0 0\n",
    "        #\n",
    "        # Where \"type_ids\" are used to indicate whether this is the first\n",
    "        # sequence or the second sequence. The embedding vectors for `type=0` and\n",
    "        # `type=1` were learned during pre-training and are added to the wordpiece\n",
    "        # embedding vector (and position vector). This is not *strictly* necessary\n",
    "        # since the [SEP] token unambigiously separates the sequences, but it makes\n",
    "        # it easier for the model to learn the concept of sequences.\n",
    "        #\n",
    "        # For classification tasks, the first vector (corresponding to [CLS]) is\n",
    "        # used as as the \"sentence vector\". Note that this only makes sense because\n",
    "        # the entire model is fine-tuned.\n",
    "        tokens = [\"[CLS]\"] + tokens + [\"[SEP]\"]\n",
    "        segment_ids = [0] * len(tokens)\n",
    "\n",
    "        input_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "\n",
    "        # The mask has 1 for real tokens and 0 for padding tokens. Only real\n",
    "        # tokens are attended to.\n",
    "        input_mask = [1] * len(input_ids)\n",
    "\n",
    "        # Zero-pad up to the sequence length.\n",
    "        padding = [0] * (max_seq_length - len(input_ids))\n",
    "        input_ids += padding\n",
    "        input_mask += padding\n",
    "        segment_ids += padding\n",
    "\n",
    "        assert len(input_ids) == max_seq_length\n",
    "        assert len(input_mask) == max_seq_length\n",
    "        assert len(segment_ids) == max_seq_length\n",
    "\n",
    "        if ex_index < 5:\n",
    "            logger.info(\"*** Example ***\")\n",
    "            logger.info(\"guid: %s\" % (example.guid))\n",
    "            logger.info(\"tokens: %s\" % \" \".join(\n",
    "                    [str(x) for x in tokens]))\n",
    "            logger.info(\"input_ids: %s\" % \" \".join([str(x) for x in input_ids]))\n",
    "            logger.info(\"input_mask: %s\" % \" \".join([str(x) for x in input_mask]))\n",
    "            logger.info(\n",
    "                    \"segment_ids: %s\" % \" \".join([str(x) for x in segment_ids]))\n",
    "            logger.info(\"target: %s\" % (example.target))\n",
    "\n",
    "        features.append(\n",
    "                InputFeatures(input_ids=input_ids,\n",
    "                              input_mask=input_mask,\n",
    "                              segment_ids=segment_ids,\n",
    "                              target=example.target))\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FreezableBertAdam(BertAdam):\n",
    "    def get_lr(self):\n",
    "        lr = []\n",
    "        for group in self.param_groups:\n",
    "            for p in group['params']:\n",
    "                state = self.state[p]\n",
    "                if len(state) == 0:\n",
    "                    continue\n",
    "                if group['t_total'] != -1:\n",
    "                    schedule_fct = SCHEDULES[group['schedule']]\n",
    "                    lr_scheduled = group['lr'] * schedule_fct(state['step']/group['t_total'], group['warmup'])\n",
    "                else:\n",
    "                    lr_scheduled = group['lr']\n",
    "                lr.append(lr_scheduled)\n",
    "        return lr    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def children(m):\n",
    "    return m if isinstance(m, (list, tuple)) else list(m.children())\n",
    "\n",
    "\n",
    "def set_trainable_attr(m, b):\n",
    "    m.trainable = b\n",
    "    for p in m.parameters():\n",
    "        p.requires_grad = b\n",
    "\n",
    "\n",
    "def apply_leaf(m, f):\n",
    "    c = children(m)\n",
    "    if isinstance(m, nn.Module):\n",
    "        f(m)\n",
    "    if len(c) > 0:\n",
    "        for l in c:\n",
    "            apply_leaf(l, f)\n",
    "\n",
    "\n",
    "def set_trainable(l, b):\n",
    "    apply_leaf(l, lambda m: set_trainable_attr(m, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_model_parameters(model):\n",
    "    logger.info(\n",
    "        \"# of paramters: {:,d}\".format(\n",
    "            sum(p.numel() for p in model.parameters())))\n",
    "    logger.info(\n",
    "        \"# of trainable paramters: {:,d}\".format(\n",
    "            sum(p.numel() for p in model.parameters() if p.requires_grad)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained(\n",
    "    \"bert-base-uncased\", do_lower_case=True, \n",
    "    cache_dir=PYTORCH_PRETRAINED_BERT_CACHE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_optimizer(num_train_optimization_steps: int, learning_rate: float):\n",
    "    grouped_parameters = [\n",
    "       x for x in optimizer_grouped_parameters if any([p.requires_grad for p in x[\"params\"]])\n",
    "    ]\n",
    "    for group in grouped_parameters:\n",
    "        group['lr'] = learning_rate\n",
    "    if FP16:\n",
    "        try:\n",
    "            from apex.optimizers import FP16_Optimizer\n",
    "            from apex.optimizers import FusedAdam\n",
    "        except ImportError:\n",
    "            raise ImportError(\"Please install apex from https://www.github.com/nvidia/apex \"\n",
    "                              \"to use distributed and fp16 training.\")\n",
    "\n",
    "        optimizer = FusedAdam(grouped_parameters,\n",
    "                              lr=learning_rate, bias_correction=False,\n",
    "                              max_grad_norm=1.0)\n",
    "        if args.loss_scale == 0:\n",
    "            optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)\n",
    "        else:\n",
    "            optimizer = FP16_Optimizer(optimizer, static_loss_scale=LOSS_SCALE)\n",
    "\n",
    "    else:\n",
    "        optimizer = FreezableBertAdam(grouped_parameters,\n",
    "                             lr=learning_rate, warmup=WARMUP_PROPORTION,\n",
    "                             t_total=num_train_optimization_steps)\n",
    "    return optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model: nn.Module, num_epochs: int, learning_rate: float):\n",
    "    num_train_optimization_steps = len(train_dataloader) * num_epochs \n",
    "    optimizer = get_optimizer(num_train_optimization_steps, learning_rate)\n",
    "    assert all([x[\"lr\"] == learning_rate for x in optimizer.param_groups])\n",
    "    global_step = 0\n",
    "    nb_tr_steps = 0\n",
    "    tr_loss = 0\n",
    "    logger.info(\"***** Running training *****\")\n",
    "    logger.info(\"  Num examples = %d\", len(train_features))\n",
    "    logger.info(\"  Batch size = %d\", BATCH_SIZE)\n",
    "    logger.info(\"  Num steps = %d\", num_train_optimization_steps)    \n",
    "    model.train()\n",
    "    mb = master_bar(range(num_epochs))\n",
    "    tr_loss = 0\n",
    "    nb_tr_examples, nb_tr_steps = 0, 0    \n",
    "    for _ in mb:\n",
    "        for step, batch in enumerate(progress_bar(train_dataloader, parent=mb)):\n",
    "            batch = tuple(t.to(device) for t in batch)\n",
    "            input_ids, input_mask, segment_ids, target = batch\n",
    "            loss = model(input_ids, segment_ids, input_mask, target)\n",
    "            if n_gpu > 1:\n",
    "                loss = loss.mean() # mean() to average on multi-gpu.\n",
    "\n",
    "            if FP16:\n",
    "                optimizer.backward(loss)\n",
    "            else:\n",
    "                loss.backward()\n",
    "\n",
    "            if tr_loss == 0:\n",
    "                tr_loss = loss.item()\n",
    "            else:\n",
    "                tr_loss = tr_loss * 0.9 + loss.item() * 0.1\n",
    "            nb_tr_examples += input_ids.size(0)\n",
    "            nb_tr_steps += 1\n",
    "            if FP16:\n",
    "                # modify learning rate with special warm up BERT uses\n",
    "                # if args.fp16 is False, BertAdam is used that handles this automatically\n",
    "                lr_this_step = (\n",
    "                     LR * warmup_linear(global_step/num_train_optimization_steps, WARMUP_PROPORTION))\n",
    "                for param_group in optimizer.param_groups:\n",
    "                    param_group['lr'] = lr_this_step\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "            global_step += 1\n",
    "            mb.child.comment = f'loss: {tr_loss:.4f} lr: {optimizer.get_lr()[0]:.2E}'\n",
    "    logger.info(\"  train loss = %.4f\", tr_loss) \n",
    "    return tr_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_examples = MISLMacroProcessor(\"Char\").get_train_examples() # Change the argument passed to MISLMacroProcessor to whichever MISL element you would like to score. These are the columns of the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features = convert_examples_to_features(\n",
    "    train_examples, MAX_SEQ_LENGTH, tokenizer)\n",
    "del train_examples\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare model\n",
    "model = BertForSequenceRegression.from_pretrained(\n",
    "    \"bert-base-uncased\",\n",
    "    cache_dir=PYTORCH_PRETRAINED_BERT_CACHE)\n",
    "if FP16:\n",
    "    model.half()\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare optimizer\n",
    "param_optimizer = list(model.named_parameters())\n",
    "no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n",
    "optimizer_grouped_parameters = [\n",
    "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n",
    "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)\n",
    "all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)\n",
    "all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)\n",
    "all_targets = torch.tensor([f.target for f in train_features], dtype=torch.float)\n",
    "train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_targets)\n",
    "train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train only the \"pooler\" and the final linear layer\n",
    "set_trainable(model, True)\n",
    "set_trainable(model.bert.embeddings, False)\n",
    "set_trainable(model.bert.encoder, False)\n",
    "count_model_parameters(model)\n",
    "train(model, num_epochs = 16, learning_rate = 5e-4) # The number of epochs and learning rate varied depending on the element being scored. The hyperparameters for each element are provided below.\n",
    "\n",
    "# Char: num_epochs = 16, learning_rate = 5e-4\n",
    "# Sett: num_epochs = 16, learning_rate = 5e-4\n",
    "# IE: num_epochs = 16, learning_rate = 1e-3\n",
    "# Plan: num_epochs = 16, learning_rate = 1e-3\n",
    "# Act: num_epochs = 16, learning_rate = 1e-3\n",
    "# Con: num_epochs = 16, learning_rate = 5e-4\n",
    "# ENP: num_epochs = 16, learning_rate = 1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save a trained model\n",
    "model_to_save = model.module if hasattr(model, 'module') else model  \n",
    "output_model_file = \"./regressor_stage1.pth\"\n",
    "# torch.save(model_to_save.state_dict(), output_model_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the last two layer, too\n",
    "set_trainable(model.bert.encoder.layer[11], True)\n",
    "set_trainable(model.bert.encoder.layer[10], True)\n",
    "count_model_parameters(model)\n",
    "train(model, num_epochs = 8, learning_rate = 5e-5) # The number of epochs and learning rate varied depending on the element being scored. The hyperparameters for each element are provided below.\n",
    "\n",
    "# Char: num_epochs = 16, learning_rate = 5e-5\n",
    "# Sett: num_epochs = 16, learning_rate = 5e-5\n",
    "# IE: num_epochs = 8, learning_rate = 5e-5\n",
    "# Plan: num_epochs = 8, learning_rate = 5e-5\n",
    "# Act: num_epochs = 8, learning_rate = 5e-5\n",
    "# Con: num_epochs = 8, learning_rate = 1e-6\n",
    "# ENP: num_epochs = 8, learning_rate = 5e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save a trained model\n",
    "model_to_save = model.module if hasattr(model, 'module') else model  \n",
    "output_model_file = \"./regressor_stage2.pth\"\n",
    "# torch.save(model_to_save.state_dict(), output_model_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train all layers\n",
    "set_trainable(model, True)\n",
    "count_model_parameters(model)\n",
    "train(model, num_epochs = 8, learning_rate = 1e-5) # The number of epochs and learning rate varied depending on the element being scored. The hyperparameters for each element are provided below.\n",
    "\n",
    "# Char: num_epochs = 16, learning_rate = 5e-5\n",
    "# Sett: num_epochs = 16, learning_rate = 1e-5\n",
    "# IE: num_epochs = 8, learning_rate = 1e-5\n",
    "# Plan: num_epochs = 8, learning_rate = 1e-5\n",
    "# Act: num_epochs = 8, learning_rate = 1e-5\n",
    "# Con: num_epochs = 8, learning_rate = 1e-7\n",
    "# ENP: num_epochs = 8, learning_rate = 1e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save a trained model\n",
    "model_to_save = model.module if hasattr(model, 'module') else model  \n",
    "output_model_file = \"./regressor_stage3.pth\"\n",
    "# torch.save(model_to_save.state_dict(), output_model_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train_features\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_examples = MISLMacroProcessor(\"Char\").get_dev_examples() # Set the argument to the MISLMacroProcessor to whatever element the model was trained on\n",
    "eval_features = convert_examples_to_features(\n",
    "    eval_examples, MAX_SEQ_LENGTH, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger.info(\"***** Running evaluation *****\")\n",
    "logger.info(\"  Num examples = %d\", len(eval_examples))\n",
    "logger.info(\"  Batch size = %d\", BATCH_SIZE * 5)\n",
    "all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)\n",
    "all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)\n",
    "all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)\n",
    "all_targets = torch.tensor([f.target for f in eval_features], dtype=torch.float)\n",
    "eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_targets)\n",
    "# Run prediction for full data\n",
    "eval_sampler = SequentialSampler(eval_data)\n",
    "eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=BATCH_SIZE * 5)\n",
    "\n",
    "model.eval()\n",
    "eval_loss, eval_accuracy = 0, 0\n",
    "nb_eval_steps, nb_eval_examples = 0, 0\n",
    "\n",
    "mb = progress_bar(eval_dataloader)\n",
    "pred = []\n",
    "real = []\n",
    "for input_ids, input_mask, segment_ids, targets in mb:\n",
    "    input_ids = input_ids.to(device)\n",
    "    input_mask = input_mask.to(device)\n",
    "    segment_ids = segment_ids.to(device)\n",
    "    targets = targets.to(device)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        tmp_eval_loss = model(input_ids, segment_ids, input_mask, targets)\n",
    "        outputs = model(input_ids, segment_ids, input_mask)\n",
    "\n",
    "    outputs = [item for sublist in np.round(outputs.detach().cpu().numpy(),0).astype(int).tolist() for item in sublist]\n",
    "    targets = np.round(targets.to('cpu').numpy(),0).astype(int).tolist()\n",
    "    pred.extend(outputs)\n",
    "    real.extend(targets)\n",
    "    # tmp_eval_accuracy = accuracy(logits, label_ids)\n",
    "\n",
    "    eval_loss += tmp_eval_loss.mean().item()\n",
    "    # eval_accuracy += tmp_eval_accuracy\n",
    "\n",
    "    nb_eval_examples += input_ids.size(0)\n",
    "    nb_eval_steps += 1\n",
    "    mb.comment = f'{eval_loss / nb_eval_steps:.4f}'\n",
    "\n",
    "eval_loss / nb_eval_steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate QWK on test set\n",
    "metrics.quadratic_weighted_kappa(pred, real, max_rating=3, min_rating=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compare predictions to Expert Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "expert_df = pd.read_csv(\"/home/.../ExpertScores.csv\") # Change to path of expert scored data\n",
    "texts = expert_df['vecOfNarratives'].values\n",
    "scores = expert_df['Char'].values # Set the selected column of the expert data frame to whatever element BERT was trained on.\n",
    "examples = []\n",
    "for (i, (texts, target)) in enumerate(zip(texts, scores)):\n",
    "    examples.append(InputExample(guid=i, text=texts, target=target))\n",
    "    \n",
    "test_features = convert_examples_to_features(\n",
    "    examples, MAX_SEQ_LENGTH, tokenizer)\n",
    "#del examples\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger.info(\"***** Running Testing *****\")\n",
    "logger.info(\"  Num examples = %d\", len(examples))\n",
    "logger.info(\"  Batch size = %d\", BATCH_SIZE * 5)\n",
    "all_input_ids = torch.tensor([f.input_ids for f in test_features], dtype=torch.long)\n",
    "all_input_mask = torch.tensor([f.input_mask for f in test_features], dtype=torch.long)\n",
    "all_segment_ids = torch.tensor([f.segment_ids for f in test_features], dtype=torch.long)\n",
    "all_targets = torch.tensor([f.target for f in test_features], dtype=torch.float)\n",
    "test_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_targets)\n",
    "# Run prediction for full data\n",
    "test_sampler = SequentialSampler(test_data)\n",
    "test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=BATCH_SIZE * 5)\n",
    "\n",
    "model.eval()\n",
    "test_loss, test_accuracy = 0, 0\n",
    "nb_test_steps, nb_test_examples = 0, 0\n",
    "\n",
    "mb = progress_bar(test_dataloader)\n",
    "pred = []\n",
    "real = []\n",
    "for input_ids, input_mask, segment_ids, targets in mb:\n",
    "    input_ids = input_ids.to(device)\n",
    "    input_mask = input_mask.to(device)\n",
    "    segment_ids = segment_ids.to(device)\n",
    "    targets = targets.to(device)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        tmp_test_loss = model(input_ids, segment_ids, input_mask, targets)\n",
    "        outputs = model(input_ids, segment_ids, input_mask)\n",
    "\n",
    "    outputs = [item for sublist in np.round(outputs.detach().cpu().numpy(),0).astype(int).tolist() for item in sublist]\n",
    "    targets = np.round(targets.to('cpu').numpy(),0).astype(int).tolist()\n",
    "    pred.extend(outputs)\n",
    "    real.extend(targets)\n",
    "    # tmp_eval_accuracy = accuracy(logits, label_ids)\n",
    "\n",
    "    test_loss += tmp_test_loss.mean().item()\n",
    "    # eval_accuracy += tmp_eval_accuracy\n",
    "\n",
    "    nb_test_examples += input_ids.size(0)\n",
    "    nb_test_steps += 1\n",
    "    mb.comment = f'{eval_loss / nb_eval_steps:.4f}'\n",
    "\n",
    "test_loss / nb_test_steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate QWK between predictions and expert scores\n",
    "metrics.quadratic_weighted_kappa(pred, real, max_rating=3, min_rating=0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}