QUEST_BERT(HuggingFace)_Solution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "#Import Statements\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "import os\n",
    "from math import floor, ceil\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "from sklearn.model_selection import GroupKFold\n",
    "from scipy.stats import spearmanr\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "import tensorflow.keras.backend as K\n",
    "\n",
    "from transformers import BertTokenizer\n",
    "from transformers import BertConfig\n",
    "from transformers import TFBertModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "#*===================================================Read data and tokenizer==============================================*\n",
    "\n",
    "#Read tokenizer and data, as well as defining the maximum sequence length that will be used for the input to Bert (maximum is usually 512 tokens)\n",
    "\n",
    "PATH = '../input/google-quest-challenge/'\n",
    "\n",
    "BERT_PATH = '../input/bert-base-uncased-huggingface-transformer/'\n",
    "tokenizer = BertTokenizer.from_pretrained(BERT_PATH+'bert-base-uncased-vocab.txt')\n",
    "\n",
    "MAX_SEQUENCE_LENGTH = 384\n",
    "\n",
    "df_train = pd.read_csv(PATH+'train.csv')\n",
    "df_test = pd.read_csv(PATH+'test.csv')\n",
    "df_sub = pd.read_csv(PATH+'sample_submission.csv')\n",
    "print('train shape =', df_train.shape)\n",
    "print('test shape =', df_test.shape)\n",
    "\n",
    "output_categories = list(df_train.columns[11:])\n",
    "input_categories = list(df_train.columns[[1,2,5]])\n",
    "print('\\noutput categories:\\n\\t', output_categories)\n",
    "print('\\ninput categories:\\n\\t', input_categories)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Observations**\n",
    "\n",
    "1) Our train dataset consists of 6079 rows and test dataset consists of 476 rows.\n",
    "\n",
    "2) We have total 30 output labels and 3 input columns i.e. Question Title, Question Body and Answer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#*======================================================Preprocessing Functions========================================================*\n",
    "\n",
    "# These are some functions that will be used to preprocess the raw text data into useable Bert inputs.\n",
    "\n",
    "def _convert_to_transformer_inputs(title, question, answer, tokenizer, max_sequence_length):\n",
    "    \"\"\"Converts tokenized input to ids, masks and segments for transformer (including bert)\"\"\"\n",
    "    \n",
    "    def return_id(str1, str2, truncation_strategy, length):\n",
    "\n",
    "        inputs = tokenizer.encode_plus(str1, str2,\n",
    "            add_special_tokens=True,\n",
    "            max_length=length,\n",
    "            truncation_strategy=truncation_strategy)\n",
    "        \n",
    "        input_ids =  inputs[\"input_ids\"]\n",
    "        input_masks = [1] * len(input_ids)\n",
    "        input_segments = inputs[\"token_type_ids\"]\n",
    "        padding_length = length - len(input_ids)\n",
    "        padding_id = tokenizer.pad_token_id\n",
    "        input_ids = input_ids + ([padding_id] * padding_length)\n",
    "        input_masks = input_masks + ([0] * padding_length)\n",
    "        input_segments = input_segments + ([0] * padding_length)\n",
    "        \n",
    "        return [input_ids, input_masks, input_segments]\n",
    "    \n",
    "    input_ids_q, input_masks_q, input_segments_q = return_id(\n",
    "        title + ' ' + question, None, 'longest_first', max_sequence_length)\n",
    "    \n",
    "    input_ids_a, input_masks_a, input_segments_a = return_id(\n",
    "        answer, None, 'longest_first', max_sequence_length)\n",
    "    \n",
    "    return [input_ids_q, input_masks_q, input_segments_q,\n",
    "            input_ids_a, input_masks_a, input_segments_a]\n",
    "\n",
    "def compute_input_arrays(df, columns, tokenizer, max_sequence_length):\n",
    "    input_ids_q, input_masks_q, input_segments_q = [], [], []\n",
    "    input_ids_a, input_masks_a, input_segments_a = [], [], []\n",
    "    for _, instance in tqdm(df[columns].iterrows()):\n",
    "        t, q, a = instance.question_title, instance.question_body, instance.answer\n",
    "\n",
    "        ids_q, masks_q, segments_q, ids_a, masks_a, segments_a = \\\n",
    "        _convert_to_transformer_inputs(t, q, a, tokenizer, max_sequence_length)\n",
    "        \n",
    "        input_ids_q.append(ids_q)\n",
    "        input_masks_q.append(masks_q)\n",
    "        input_segments_q.append(segments_q)\n",
    "\n",
    "        input_ids_a.append(ids_a)\n",
    "        input_masks_a.append(masks_a)\n",
    "        input_segments_a.append(segments_a)\n",
    "        \n",
    "    return [np.asarray(input_ids_q, dtype=np.int32), \n",
    "            np.asarray(input_masks_q, dtype=np.int32), \n",
    "            np.asarray(input_segments_q, dtype=np.int32),\n",
    "            np.asarray(input_ids_a, dtype=np.int32), \n",
    "            np.asarray(input_masks_a, dtype=np.int32), \n",
    "            np.asarray(input_segments_a, dtype=np.int32)]\n",
    "\n",
    "def compute_output_arrays(df, columns):\n",
    "    return np.asarray(df[columns])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Observations:**\n",
    "\n",
    "1) _convert_to_transformer_inputs is a generic method to convert our dataset into equivalent input that any model of transformer library is expecting.\n",
    "\n",
    "2) Question body and Question Title are merged into one and Answer is taken as a separate segment.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#*============================================Model Creation and Metric Calculation=======================================*\n",
    "\n",
    "def compute_spearmanr_ignore_nan(trues, preds):\n",
    "    rhos = []\n",
    "    for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)):\n",
    "        rhos.append(spearmanr(tcol, pcol).correlation)\n",
    "    return np.nanmean(rhos)\n",
    "\n",
    "\n",
    "q_id = tf.keras.layers.Input((MAX_SEQUENCE_LENGTH,), dtype=tf.int32)\n",
    "a_id = tf.keras.layers.Input((MAX_SEQUENCE_LENGTH,), dtype=tf.int32)\n",
    "\n",
    "q_mask = tf.keras.layers.Input((MAX_SEQUENCE_LENGTH,), dtype=tf.int32)\n",
    "a_mask = tf.keras.layers.Input((MAX_SEQUENCE_LENGTH,), dtype=tf.int32)\n",
    "\n",
    "q_atn = tf.keras.layers.Input((MAX_SEQUENCE_LENGTH,), dtype=tf.int32)\n",
    "a_atn = tf.keras.layers.Input((MAX_SEQUENCE_LENGTH,), dtype=tf.int32)\n",
    "\n",
    "\n",
    "def create_model():\n",
    "    \n",
    "    config = BertConfig() # print(config) to see settings\n",
    "    config.output_hidden_states = False # Set to True to obtain hidden states\n",
    "    # caution: when using e.g. XLNet, XLNetConfig() will automatically use xlnet-large config\n",
    "    \n",
    "    # normally \".from_pretrained('bert-base-uncased')\", but because of no internet, the \n",
    "    # pretrained model has been downloaded manually and uploaded to kaggle. \n",
    "    bert_model = TFBertModel.from_pretrained(\n",
    "        BERT_PATH+'bert-base-uncased-tf_model.h5', config=config)\n",
    "    \n",
    "    # if config.output_hidden_states = True, obtain hidden states via bert_model(...)[-1]\n",
    "    q_embedding = bert_model(q_id, attention_mask=q_mask, token_type_ids=q_atn)[0]\n",
    "    a_embedding = bert_model(a_id, attention_mask=a_mask, token_type_ids=a_atn)[0]\n",
    "    \n",
    "    q = tf.keras.layers.GlobalAveragePooling1D()(q_embedding)\n",
    "    a = tf.keras.layers.GlobalAveragePooling1D()(a_embedding)\n",
    "    \n",
    "    x = tf.keras.layers.Concatenate()([q, a])\n",
    "    \n",
    "    x = tf.keras.layers.Dropout(0.2)(x)\n",
    "    \n",
    "    x = tf.keras.layers.Dense(30, activation='sigmoid')(x)\n",
    "\n",
    "    model = tf.keras.models.Model(inputs=[q_id, q_mask, q_atn, a_id, a_mask, a_atn,], outputs=x)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Observations:**\n",
    "\n",
    "1) compute_spearmanr_ignore_nan is used to compute the competition metric for the validation set\n",
    "\n",
    "2) create_model contains the actual architecture that will be used to finetune BERT to our dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#*=====================================Generating Input and Output for Traina & Test=======================================*\n",
    "\n",
    "outputs = compute_output_arrays(df_train, output_categories)\n",
    "inputs = compute_input_arrays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)\n",
    "test_inputs = compute_input_arrays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#*===============================================Training, Validation and Testing==========================================*\n",
    "\n",
    "gkf = GroupKFold(n_splits=5).split(X=df_train.question_body, groups=df_train.question_body)\n",
    "\n",
    "valid_preds = []\n",
    "test_preds = []\n",
    "for fold, (train_idx, valid_idx) in enumerate(gkf):\n",
    "    \n",
    "    # will actually only do 2 folds (out of 5) to manage < 2h\n",
    "    if fold in [0, 2]:\n",
    "\n",
    "        train_inputs = [inputs[i][train_idx] for i in range(len(inputs))]\n",
    "        train_outputs = outputs[train_idx]\n",
    "\n",
    "        valid_inputs = [inputs[i][valid_idx] for i in range(len(inputs))]\n",
    "        valid_outputs = outputs[valid_idx]\n",
    "        \n",
    "        K.clear_session()\n",
    "        model = create_model()\n",
    "        optimizer = tf.keras.optimizers.Adam(learning_rate=2e-5)\n",
    "        model.compile(loss='binary_crossentropy', optimizer=optimizer)\n",
    "        model.fit(train_inputs, train_outputs, epochs=3, batch_size=6)\n",
    "        # model.save_weights(f'bert-{fold}.h5')\n",
    "        valid_preds.append(model.predict(valid_inputs))\n",
    "        test_preds.append(model.predict(test_inputs))\n",
    "        \n",
    "        rho_val = compute_spearmanr_ignore_nan(valid_outputs, valid_preds[-1])\n",
    "        print('validation score = ', rho_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Observations:**\n",
    "\n",
    "1) Loops over the folds in gkf and trains each fold for 3 epochs with a learning rate of 3e-5 and batch_size of 6.\n",
    "\n",
    "2) A simple binary crossentropy is used as the objective loss-function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#*===============================================Process and Submit Test Predictions==========================================*\n",
    "\n",
    "df_sub.iloc[:, 1:] = np.average(test_preds, axis=0)\n",
    "\n",
    "df_sub.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Observations:**\n",
    "\n",
    "Average fold predictions, then save as submission.csv"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}