191228_Baseline_with_validation_Cara.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import GroupKFold\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm.notebook import tqdm\n",
    "import tensorflow_hub as hub\n",
    "import tensorflow as tf\n",
    "import bert.tokenization as tokenization\n",
    "import tensorflow.keras.backend as K\n",
    "import gc\n",
    "import os\n",
    "from scipy.stats import spearmanr\n",
    "from math import floor, ceil\n",
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "np.set_printoptions(suppress=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following cell is for bert_tokenization package"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import re\n",
    "import unicodedata\n",
    "import six\n",
    "import tensorflow as tf\n",
    "\n",
    "\n",
    "def validate_case_matches_checkpoint(do_lower_case, init_checkpoint):\n",
    "    \"\"\"Checks whether the casing config is consistent with the checkpoint name.\"\"\"\n",
    "\n",
    "    # The casing has to be passed in by the user and there is no explicit check\n",
    "    # as to whether it matches the checkpoint. The casing information probably\n",
    "    # should have been stored in the bert_config.json file, but it's not, so\n",
    "    # we have to heuristically detect it to validate.\n",
    "\n",
    "    if not init_checkpoint:\n",
    "        return\n",
    "\n",
    "    m = re.match(\"^.*?([A-Za-z0-9_-]+)/bert_model.ckpt\", init_checkpoint)\n",
    "    if m is None:\n",
    "        return\n",
    "\n",
    "    model_name = m.group(1)\n",
    "\n",
    "    lower_models = [\n",
    "        \"uncased_L-24_H-1024_A-16\", \"uncased_L-12_H-768_A-12\",\n",
    "        \"multilingual_L-12_H-768_A-12\", \"chinese_L-12_H-768_A-12\"\n",
    "    ]\n",
    "\n",
    "    cased_models = [\n",
    "        \"cased_L-12_H-768_A-12\", \"cased_L-24_H-1024_A-16\",\n",
    "        \"multi_cased_L-12_H-768_A-12\"\n",
    "    ]\n",
    "\n",
    "    is_bad_config = False\n",
    "    if model_name in lower_models and not do_lower_case:\n",
    "        is_bad_config = True\n",
    "        actual_flag = \"False\"\n",
    "        case_name = \"lowercased\"\n",
    "        opposite_flag = \"True\"\n",
    "\n",
    "    if model_name in cased_models and do_lower_case:\n",
    "        is_bad_config = True\n",
    "        actual_flag = \"True\"\n",
    "        case_name = \"cased\"\n",
    "        opposite_flag = \"False\"\n",
    "\n",
    "    if is_bad_config:\n",
    "        raise ValueError(\n",
    "            \"You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`. \"\n",
    "            \"However, `%s` seems to be a %s model, so you \"\n",
    "            \"should pass in `--do_lower_case=%s` so that the fine-tuning matches \"\n",
    "            \"how the model was pre-training. If this error is wrong, please \"\n",
    "            \"just comment out this check.\" % (actual_flag, init_checkpoint,\n",
    "                                              model_name, case_name, opposite_flag))\n",
    "\n",
    "\n",
    "def convert_to_unicode(text):\n",
    "    \"\"\"Converts `text` to Unicode (if it's not already), assuming utf-8 input.\"\"\"\n",
    "    if six.PY3:\n",
    "        if isinstance(text, str):\n",
    "            return text\n",
    "        elif isinstance(text, bytes):\n",
    "            return text.decode(\"utf-8\", \"ignore\")\n",
    "        else:\n",
    "            raise ValueError(\"Unsupported string type: %s\" % (type(text)))\n",
    "    elif six.PY2:\n",
    "        if isinstance(text, str):\n",
    "            return text.decode(\"utf-8\", \"ignore\")\n",
    "        elif isinstance(text, unicode):\n",
    "            return text\n",
    "        else:\n",
    "            raise ValueError(\"Unsupported string type: %s\" % (type(text)))\n",
    "    else:\n",
    "        raise ValueError(\"Not running on Python2 or Python 3?\")\n",
    "\n",
    "\n",
    "def printable_text(text):\n",
    "    \"\"\"Returns text encoded in a way suitable for print or `tf.logging`.\"\"\"\n",
    "\n",
    "    # These functions want `str` for both Python2 and Python3, but in one case\n",
    "    # it's a Unicode string and in the other it's a byte string.\n",
    "    if six.PY3:\n",
    "        if isinstance(text, str):\n",
    "            return text\n",
    "        elif isinstance(text, bytes):\n",
    "            return text.decode(\"utf-8\", \"ignore\")\n",
    "        else:\n",
    "            raise ValueError(\"Unsupported string type: %s\" % (type(text)))\n",
    "    elif six.PY2:\n",
    "        if isinstance(text, str):\n",
    "            return text\n",
    "        elif isinstance(text, unicode):\n",
    "            return text.encode(\"utf-8\")\n",
    "        else:\n",
    "            raise ValueError(\"Unsupported string type: %s\" % (type(text)))\n",
    "    else:\n",
    "        raise ValueError(\"Not running on Python2 or Python 3?\")\n",
    "\n",
    "\n",
    "def load_vocab(vocab_file):\n",
    "    \"\"\"Loads a vocabulary file into a dictionary.\"\"\"\n",
    "    vocab = collections.OrderedDict()\n",
    "    index = 0\n",
    "    with tf.io.gfile.GFile(vocab_file, \"r\") as reader:\n",
    "        while True:\n",
    "            token = convert_to_unicode(reader.readline())\n",
    "            if not token:\n",
    "                break\n",
    "            token = token.strip()\n",
    "            vocab[token] = index\n",
    "            index += 1\n",
    "    return vocab\n",
    "\n",
    "\n",
    "def convert_by_vocab(vocab, items):\n",
    "    \"\"\"Converts a sequence of [tokens|ids] using the vocab.\"\"\"\n",
    "    output = []\n",
    "    for item in items:\n",
    "        output.append(vocab[item])\n",
    "    return output\n",
    "\n",
    "\n",
    "def convert_tokens_to_ids(vocab, tokens):\n",
    "    return convert_by_vocab(vocab, tokens)\n",
    "\n",
    "\n",
    "def convert_ids_to_tokens(inv_vocab, ids):\n",
    "    return convert_by_vocab(inv_vocab, ids)\n",
    "\n",
    "\n",
    "def whitespace_tokenize(text):\n",
    "    \"\"\"Runs basic whitespace cleaning and splitting on a piece of text.\"\"\"\n",
    "    text = text.strip()\n",
    "    if not text:\n",
    "        return []\n",
    "    tokens = text.split()\n",
    "    return tokens\n",
    "\n",
    "\n",
    "class FullTokenizer(object):\n",
    "    \"\"\"Runs end-to-end tokenziation.\"\"\"\n",
    "\n",
    "    def __init__(self, vocab_file, do_lower_case=True):\n",
    "        self.vocab = load_vocab(vocab_file)\n",
    "        self.inv_vocab = {v: k for k, v in self.vocab.items()}\n",
    "        self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case)\n",
    "        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)\n",
    "\n",
    "    def tokenize(self, text):\n",
    "        split_tokens = []\n",
    "        for token in self.basic_tokenizer.tokenize(text):\n",
    "            for sub_token in self.wordpiece_tokenizer.tokenize(token):\n",
    "                split_tokens.append(sub_token)\n",
    "\n",
    "        return split_tokens\n",
    "\n",
    "    def convert_tokens_to_ids(self, tokens):\n",
    "        return convert_by_vocab(self.vocab, tokens)\n",
    "\n",
    "    def convert_ids_to_tokens(self, ids):\n",
    "        return convert_by_vocab(self.inv_vocab, ids)\n",
    "\n",
    "\n",
    "class BasicTokenizer(object):\n",
    "    \"\"\"Runs basic tokenization (punctuation splitting, lower casing, etc.).\"\"\"\n",
    "\n",
    "    def __init__(self, do_lower_case=True):\n",
    "        \"\"\"Constructs a BasicTokenizer.\n",
    "        Args:\n",
    "          do_lower_case: Whether to lower case the input.\n",
    "        \"\"\"\n",
    "        self.do_lower_case = do_lower_case\n",
    "\n",
    "    def tokenize(self, text):\n",
    "        \"\"\"Tokenizes a piece of text.\"\"\"\n",
    "        text = convert_to_unicode(text)\n",
    "        text = self._clean_text(text)\n",
    "\n",
    "        # This was added on November 1st, 2018 for the multilingual and Chinese\n",
    "        # models. This is also applied to the English models now, but it doesn't\n",
    "        # matter since the English models were not trained on any Chinese data\n",
    "        # and generally don't have any Chinese data in them (there are Chinese\n",
    "        # characters in the vocabulary because Wikipedia does have some Chinese\n",
    "        # words in the English Wikipedia.).\n",
    "        text = self._tokenize_chinese_chars(text)\n",
    "\n",
    "        orig_tokens = whitespace_tokenize(text)\n",
    "        split_tokens = []\n",
    "        for token in orig_tokens:\n",
    "            if self.do_lower_case:\n",
    "                token = token.lower()\n",
    "                token = self._run_strip_accents(token)\n",
    "            split_tokens.extend(self._run_split_on_punc(token))\n",
    "\n",
    "        output_tokens = whitespace_tokenize(\" \".join(split_tokens))\n",
    "        return output_tokens\n",
    "\n",
    "    def _run_strip_accents(self, text):\n",
    "        \"\"\"Strips accents from a piece of text.\"\"\"\n",
    "        text = unicodedata.normalize(\"NFD\", text)\n",
    "        output = []\n",
    "        for char in text:\n",
    "            cat = unicodedata.category(char)\n",
    "            if cat == \"Mn\":\n",
    "                continue\n",
    "            output.append(char)\n",
    "        return \"\".join(output)\n",
    "\n",
    "    def _run_split_on_punc(self, text):\n",
    "        \"\"\"Splits punctuation on a piece of text.\"\"\"\n",
    "        chars = list(text)\n",
    "        i = 0\n",
    "        start_new_word = True\n",
    "        output = []\n",
    "        while i < len(chars):\n",
    "            char = chars[i]\n",
    "            if _is_punctuation(char):\n",
    "                output.append([char])\n",
    "                start_new_word = True\n",
    "            else:\n",
    "                if start_new_word:\n",
    "                    output.append([])\n",
    "                start_new_word = False\n",
    "                output[-1].append(char)\n",
    "            i += 1\n",
    "\n",
    "        return [\"\".join(x) for x in output]\n",
    "\n",
    "    def _tokenize_chinese_chars(self, text):\n",
    "        \"\"\"Adds whitespace around any CJK character.\"\"\"\n",
    "        output = []\n",
    "        for char in text:\n",
    "            cp = ord(char)\n",
    "            if self._is_chinese_char(cp):\n",
    "                output.append(\" \")\n",
    "                output.append(char)\n",
    "                output.append(\" \")\n",
    "            else:\n",
    "                output.append(char)\n",
    "        return \"\".join(output)\n",
    "\n",
    "    def _is_chinese_char(self, cp):\n",
    "        \"\"\"Checks whether CP is the codepoint of a CJK character.\"\"\"\n",
    "        # This defines a \"chinese character\" as anything in the CJK Unicode block:\n",
    "        #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)\n",
    "        #\n",
    "        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,\n",
    "        # despite its name. The modern Korean Hangul alphabet is a different block,\n",
    "        # as is Japanese Hiragana and Katakana. Those alphabets are used to write\n",
    "        # space-separated words, so they are not treated specially and handled\n",
    "        # like the all of the other languages.\n",
    "        if ((cp >= 0x4E00 and cp <= 0x9FFF) or  #\n",
    "                (cp >= 0x3400 and cp <= 0x4DBF) or  #\n",
    "                (cp >= 0x20000 and cp <= 0x2A6DF) or  #\n",
    "                (cp >= 0x2A700 and cp <= 0x2B73F) or  #\n",
    "                (cp >= 0x2B740 and cp <= 0x2B81F) or  #\n",
    "                (cp >= 0x2B820 and cp <= 0x2CEAF) or\n",
    "                (cp >= 0xF900 and cp <= 0xFAFF) or  #\n",
    "                (cp >= 0x2F800 and cp <= 0x2FA1F)):  #\n",
    "            return True\n",
    "\n",
    "        return False\n",
    "\n",
    "    def _clean_text(self, text):\n",
    "        \"\"\"Performs invalid character removal and whitespace cleanup on text.\"\"\"\n",
    "        output = []\n",
    "        for char in text:\n",
    "            cp = ord(char)\n",
    "            if cp == 0 or cp == 0xfffd or _is_control(char):\n",
    "                continue\n",
    "            if _is_whitespace(char):\n",
    "                output.append(\" \")\n",
    "            else:\n",
    "                output.append(char)\n",
    "        return \"\".join(output)\n",
    "\n",
    "\n",
    "class WordpieceTokenizer(object):\n",
    "    \"\"\"Runs WordPiece tokenziation.\"\"\"\n",
    "\n",
    "    def __init__(self, vocab, unk_token=\"[UNK]\", max_input_chars_per_word=200):\n",
    "        self.vocab = vocab\n",
    "        self.unk_token = unk_token\n",
    "        self.max_input_chars_per_word = max_input_chars_per_word\n",
    "\n",
    "    def tokenize(self, text):\n",
    "        \"\"\"Tokenizes a piece of text into its word pieces.\n",
    "        This uses a greedy longest-match-first algorithm to perform tokenization\n",
    "        using the given vocabulary.\n",
    "        For example:\n",
    "          input = \"unaffable\"\n",
    "          output = [\"un\", \"##aff\", \"##able\"]\n",
    "        Args:\n",
    "          text: A single token or whitespace separated tokens. This should have\n",
    "            already been passed through `BasicTokenizer.\n",
    "        Returns:\n",
    "          A list of wordpiece tokens.\n",
    "        \"\"\"\n",
    "\n",
    "        text = convert_to_unicode(text)\n",
    "\n",
    "        output_tokens = []\n",
    "        for token in whitespace_tokenize(text):\n",
    "            chars = list(token)\n",
    "            if len(chars) > self.max_input_chars_per_word:\n",
    "                output_tokens.append(self.unk_token)\n",
    "                continue\n",
    "\n",
    "            is_bad = False\n",
    "            start = 0\n",
    "            sub_tokens = []\n",
    "            while start < len(chars):\n",
    "                end = len(chars)\n",
    "                cur_substr = None\n",
    "                while start < end:\n",
    "                    substr = \"\".join(chars[start:end])\n",
    "                    if start > 0:\n",
    "                        substr = \"##\" + substr\n",
    "                    if substr in self.vocab:\n",
    "                        cur_substr = substr\n",
    "                        break\n",
    "                    end -= 1\n",
    "                if cur_substr is None:\n",
    "                    is_bad = True\n",
    "                    break\n",
    "                sub_tokens.append(cur_substr)\n",
    "                start = end\n",
    "\n",
    "            if is_bad:\n",
    "                output_tokens.append(self.unk_token)\n",
    "            else:\n",
    "                output_tokens.extend(sub_tokens)\n",
    "        return output_tokens\n",
    "\n",
    "\n",
    "def _is_whitespace(char):\n",
    "    \"\"\"Checks whether `chars` is a whitespace character.\"\"\"\n",
    "    # \\t, \\n, and \\r are technically contorl characters but we treat them\n",
    "    # as whitespace since they are generally considered as such.\n",
    "    if char == \" \" or char == \"\\t\" or char == \"\\n\" or char == \"\\r\":\n",
    "        return True\n",
    "    cat = unicodedata.category(char)\n",
    "    if cat == \"Zs\":\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "\n",
    "def _is_control(char):\n",
    "    \"\"\"Checks whether `chars` is a control character.\"\"\"\n",
    "    # These are technically control characters but we count them as whitespace\n",
    "    # characters.\n",
    "    if char == \"\\t\" or char == \"\\n\" or char == \"\\r\":\n",
    "        return False\n",
    "    cat = unicodedata.category(char)\n",
    "    if cat in (\"Cc\", \"Cf\"):\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "\n",
    "def _is_punctuation(char):\n",
    "    \"\"\"Checks whether `chars` is a punctuation character.\"\"\"\n",
    "    cp = ord(char)\n",
    "    # We treat all non-letter/number ASCII as punctuation.\n",
    "    # Characters such as \"^\", \"$\", and \"`\" are not in the Unicode\n",
    "    # Punctuation class but we treat them as punctuation anyways, for\n",
    "    # consistency.\n",
    "    if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or\n",
    "            (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)):\n",
    "        return True\n",
    "    cat = unicodedata.category(char)\n",
    "    if cat.startswith(\"P\"):\n",
    "        return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bert-based Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import GroupKFold\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm.notebook import tqdm\n",
    "import tensorflow_hub as hub\n",
    "import tensorflow as tf\n",
    "#import bert_tokenization as tokenization\n",
    "import tensorflow.keras.backend as K\n",
    "import gc\n",
    "import os\n",
    "from scipy.stats import spearmanr\n",
    "from math import floor, ceil\n",
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "np.set_printoptions(suppress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train shape = (6079, 41)\n",
      "test shape = (476, 11)\n",
      "\n",
      "output categories:\n",
      "\t ['question_asker_intent_understanding', 'question_body_critical', 'question_conversational', 'question_expect_short_answer', 'question_fact_seeking', 'question_has_commonly_accepted_answer', 'question_interestingness_others', 'question_interestingness_self', 'question_multi_intent', 'question_not_really_a_question', 'question_opinion_seeking', 'question_type_choice', 'question_type_compare', 'question_type_consequence', 'question_type_definition', 'question_type_entity', 'question_type_instructions', 'question_type_procedure', 'question_type_reason_explanation', 'question_type_spelling', 'question_well_written', 'answer_helpful', 'answer_level_of_information', 'answer_plausible', 'answer_relevance', 'answer_satisfaction', 'answer_type_instructions', 'answer_type_procedure', 'answer_type_reason_explanation', 'answer_well_written']\n",
      "\n",
      "input categories:\n",
      "\t ['question_title', 'question_body', 'answer']\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "PATH = 'data/'\n",
    "BERT_PATH = 'bert/'\n",
    "tokenizer = FullTokenizer(BERT_PATH+'vocab.txt', True)\n",
    "MAX_SEQUENCE_LENGTH = 512\n",
    "\n",
    "df_train = pd.read_csv(PATH+'train.csv')\n",
    "df_test = pd.read_csv(PATH+'test.csv')\n",
    "df_sub = pd.read_csv(PATH+'sample_submission.csv')\n",
    "print('train shape =', df_train.shape)\n",
    "print('test shape =', df_test.shape)\n",
    "\n",
    "output_categories = list(df_train.columns[11:])\n",
    "input_categories = list(df_train.columns[[1,2,5]])\n",
    "print('\\noutput categories:\\n\\t', output_categories)\n",
    "print('\\ninput categories:\\n\\t', input_categories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _get_masks(tokens, max_seq_length):\n",
    "    \"\"\"Mask for padding\"\"\"\n",
    "    if len(tokens)>max_seq_length:\n",
    "        raise IndexError(\"Token length more than max seq length!\")\n",
    "    return [1]*len(tokens) + [0] * (max_seq_length - len(tokens))\n",
    "\n",
    "def _get_segments(tokens, max_seq_length):\n",
    "    \"\"\"Segments: 0 for the first sequence, 1 for the second\"\"\"\n",
    "    if len(tokens)>max_seq_length:\n",
    "        raise IndexError(\"Token length more than max seq length!\")\n",
    "    segments = []\n",
    "    first_sep = True\n",
    "    current_segment_id = 0\n",
    "    for token in tokens:\n",
    "        segments.append(current_segment_id)\n",
    "        if token == \"[SEP]\":\n",
    "            if first_sep:\n",
    "                first_sep = False \n",
    "            else:\n",
    "                current_segment_id = 1\n",
    "    return segments + [0] * (max_seq_length - len(tokens))\n",
    "\n",
    "def _get_ids(tokens, tokenizer, max_seq_length):\n",
    "    \"\"\"Token ids from Tokenizer vocab\"\"\"\n",
    "    token_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "    input_ids = token_ids + [0] * (max_seq_length-len(token_ids))\n",
    "    return input_ids\n",
    "\n",
    "def _trim_input(title, question, answer, max_sequence_length, \n",
    "                t_max_len=30, q_max_len=239, a_max_len=239):\n",
    "\n",
    "    t = tokenizer.tokenize(title)\n",
    "    q = tokenizer.tokenize(question)\n",
    "    a = tokenizer.tokenize(answer)\n",
    "    \n",
    "    t_len = len(t)\n",
    "    q_len = len(q)\n",
    "    a_len = len(a)\n",
    "\n",
    "    if (t_len+q_len+a_len+4) > max_sequence_length:\n",
    "        \n",
    "        if t_max_len > t_len:\n",
    "            t_new_len = t_len\n",
    "            a_max_len = a_max_len + floor((t_max_len - t_len)/2)\n",
    "            q_max_len = q_max_len + ceil((t_max_len - t_len)/2)\n",
    "        else:\n",
    "            t_new_len = t_max_len\n",
    "      \n",
    "        if a_max_len > a_len:\n",
    "            a_new_len = a_len \n",
    "            q_new_len = q_max_len + (a_max_len - a_len)\n",
    "        elif q_max_len > q_len:\n",
    "            a_new_len = a_max_len + (q_max_len - q_len)\n",
    "            q_new_len = q_len\n",
    "        else:\n",
    "            a_new_len = a_max_len\n",
    "            q_new_len = q_max_len\n",
    "            \n",
    "            \n",
    "        if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:\n",
    "            raise ValueError(\"New sequence length should be %d, but is %d\" \n",
    "                             % (max_sequence_length, (t_new_len+a_new_len+q_new_len+4)))\n",
    "        \n",
    "        t = t[:t_new_len]\n",
    "        q = q[:q_new_len]\n",
    "        a = a[:a_new_len]\n",
    "    \n",
    "    return t, q, a\n",
    "\n",
    "def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):\n",
    "    \"\"\"Converts tokenized input to ids, masks and segments for BERT\"\"\"\n",
    "    \n",
    "    stoken = [\"[CLS]\"] + title + [\"[SEP]\"] + question + [\"[SEP]\"] + answer + [\"[SEP]\"]\n",
    "\n",
    "    input_ids = _get_ids(stoken, tokenizer, max_sequence_length)\n",
    "    input_masks = _get_masks(stoken, max_sequence_length)\n",
    "    input_segments = _get_segments(stoken, max_sequence_length)\n",
    "\n",
    "    return [input_ids, input_masks, input_segments]\n",
    "\n",
    "def compute_input_arays(df, columns, tokenizer, max_sequence_length):\n",
    "    input_ids, input_masks, input_segments = [], [], []\n",
    "    for _, instance in tqdm(df[columns].iterrows()):\n",
    "        t, q, a = instance.question_title, instance.question_body, instance.answer\n",
    "\n",
    "        t, q, a = _trim_input(t, q, a, max_sequence_length)\n",
    "\n",
    "        ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)\n",
    "        input_ids.append(ids)\n",
    "        input_masks.append(masks)\n",
    "        input_segments.append(segments)\n",
    "        \n",
    "    return [np.asarray(input_ids, dtype=np.int32), \n",
    "            np.asarray(input_masks, dtype=np.int32), \n",
    "            np.asarray(input_segments, dtype=np.int32)]\n",
    "\n",
    "\n",
    "def compute_output_arrays(df, columns):\n",
    "    return np.asarray(df[columns])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_spearmanr(trues, preds):\n",
    "    rhos = []\n",
    "    for col_trues, col_pred in zip(trues.T, preds.T):\n",
    "        rhos.append(\n",
    "            spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)\n",
    "    return np.mean(rhos)\n",
    "\n",
    "\n",
    "class CustomCallback(tf.keras.callbacks.Callback):\n",
    "    \n",
    "    def __init__(self, valid_data, test_data, batch_size=16, fold=None):\n",
    "\n",
    "        self.valid_inputs = valid_data[0]\n",
    "        self.valid_outputs = valid_data[1]\n",
    "        self.test_inputs = test_data\n",
    "        \n",
    "        self.batch_size = batch_size\n",
    "        self.fold = fold\n",
    "        \n",
    "    def on_train_begin(self, logs={}):\n",
    "        self.valid_predictions = []\n",
    "        self.test_predictions = []\n",
    "        \n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        self.valid_predictions.append(\n",
    "            self.model.predict(self.valid_inputs, batch_size=self.batch_size))\n",
    "        \n",
    "        rho_val = compute_spearmanr(\n",
    "            self.valid_outputs, np.average(self.valid_predictions, axis=0))\n",
    "        \n",
    "        print(\"\\nvalidation rho: %.4f\" % rho_val)\n",
    "        \n",
    "        if self.fold is not None:\n",
    "            self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')\n",
    "        \n",
    "        self.test_predictions.append(\n",
    "            self.model.predict(self.test_inputs, batch_size=self.batch_size)\n",
    "        )\n",
    "\n",
    "def bert_model():\n",
    "    \n",
    "    input_word_ids = tf.keras.layers.Input(\n",
    "        (MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')\n",
    "    input_masks = tf.keras.layers.Input(\n",
    "        (MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')\n",
    "    input_segments = tf.keras.layers.Input(\n",
    "        (MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')\n",
    "    \n",
    "    bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)\n",
    "    \n",
    "    _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])\n",
    "    \n",
    "    x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)\n",
    "    x = tf.keras.layers.Dropout(0.2)(x)\n",
    "    out = tf.keras.layers.Dense(30, activation=\"sigmoid\", name=\"dense_output\")(x)\n",
    "\n",
    "    model = tf.keras.models.Model(\n",
    "        inputs=[input_word_ids, input_masks, input_segments], outputs=out)\n",
    "    \n",
    "    return model    \n",
    "        \n",
    "def train_and_predict(model, train_data, valid_data, test_data, \n",
    "                      learning_rate, epochs, batch_size, loss_function, fold):\n",
    "        \n",
    "    custom_callback = CustomCallback(\n",
    "        valid_data=(valid_data[0], valid_data[1]), \n",
    "        test_data=test_data,\n",
    "        batch_size=batch_size,\n",
    "        fold=None)\n",
    "\n",
    "    optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
    "    model.compile(loss=loss_function, optimizer=optimizer)\n",
    "    model.fit(train_data[0], train_data[1], epochs=epochs, \n",
    "              batch_size=batch_size, callbacks=[custom_callback])\n",
    "    \n",
    "    return custom_callback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46109bc775d94b3e9993edcaf3223905",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "gkf = GroupKFold(n_splits=10).split(X=df_train.question_body, groups=df_train.question_body) ############## originaln_splits=5\n",
    "\n",
    "#outputs = compute_output_arrays(df_train, output_categories)\n",
    "#inputs = compute_input_arays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)\n",
    "test_inputs = compute_input_arays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check the performance of the model on each column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "471e5fbd86524b4bb7838ff22f115793",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Split dataset for validation from the train set\n",
    "df_valid = df_train.sample(n=476)\n",
    "df_valid_y = df_valid[df_valid.columns[11:]]\n",
    "df_valid_x = df_valid[df_valid.columns[:11]]\n",
    "\n",
    "# Convert dataset to input\n",
    "valid_inputs = compute_input_arays(df_valid_x, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qa_id</th>\n",
       "      <th>question_title</th>\n",
       "      <th>question_body</th>\n",
       "      <th>question_user_name</th>\n",
       "      <th>question_user_page</th>\n",
       "      <th>answer</th>\n",
       "      <th>answer_user_name</th>\n",
       "      <th>answer_user_page</th>\n",
       "      <th>url</th>\n",
       "      <th>category</th>\n",
       "      <th>host</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2780</td>\n",
       "      <td>4429</td>\n",
       "      <td>Dimension of a quotient ring</td>\n",
       "      <td>\\n  What is the Krull dimension of $B=A[x,y,z]...</td>\n",
       "      <td>Zoey</td>\n",
       "      <td>https://math.stackexchange.com/users/234074</td>\n",
       "      <td>Your first ring $B$ is just $A[x^{\\pm1}]$, the...</td>\n",
       "      <td>Mariano Suárez-Álvarez</td>\n",
       "      <td>https://math.stackexchange.com/users/274</td>\n",
       "      <td>http://math.stackexchange.com/questions/138610...</td>\n",
       "      <td>SCIENCE</td>\n",
       "      <td>math.stackexchange.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6013</td>\n",
       "      <td>9539</td>\n",
       "      <td>Is there a way to read the columns in a shapef...</td>\n",
       "      <td>My shapefile has a number of fields, like road...</td>\n",
       "      <td>patrick</td>\n",
       "      <td>https://gis.stackexchange.com/users/733</td>\n",
       "      <td>Yes, a shapefile in this context is a feature ...</td>\n",
       "      <td>Petr Krebs</td>\n",
       "      <td>https://gis.stackexchange.com/users/434</td>\n",
       "      <td>http://gis.stackexchange.com/questions/4619/is...</td>\n",
       "      <td>TECHNOLOGY</td>\n",
       "      <td>gis.stackexchange.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2992</td>\n",
       "      <td>4770</td>\n",
       "      <td>Quantity based discount for single product in ...</td>\n",
       "      <td>I'm using latest build of Expresso Store, 2.3....</td>\n",
       "      <td>neekster</td>\n",
       "      <td>https://expressionengine.stackexchange.com/use...</td>\n",
       "      <td>Use this bulk discounts add-on, does exactly w...</td>\n",
       "      <td>Peter Lewis</td>\n",
       "      <td>https://expressionengine.stackexchange.com/use...</td>\n",
       "      <td>http://expressionengine.stackexchange.com/ques...</td>\n",
       "      <td>TECHNOLOGY</td>\n",
       "      <td>expressionengine.stackexchange.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>238</td>\n",
       "      <td>hide javascript/jquery scripts from html page?</td>\n",
       "      <td>How do I hide my javascript/jquery scripts fro...</td>\n",
       "      <td>Shaitender Singh</td>\n",
       "      <td>https://stackoverflow.com/users/192252</td>\n",
       "      <td>You can't, sorry. No matter what you do, even ...</td>\n",
       "      <td>Brian Campbell</td>\n",
       "      <td>https://stackoverflow.com/users/69755</td>\n",
       "      <td>http://stackoverflow.com/questions/1628799/hid...</td>\n",
       "      <td>STACKOVERFLOW</td>\n",
       "      <td>stackoverflow.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1827</td>\n",
       "      <td>2898</td>\n",
       "      <td>Which whois is the most frequently updated?</td>\n",
       "      <td>As the domain manager of 200+ domains, it's es...</td>\n",
       "      <td>Miss M</td>\n",
       "      <td>https://webmasters.stackexchange.com/users/52761</td>\n",
       "      <td>Being one who has a similar site, I would have...</td>\n",
       "      <td>closetnoc</td>\n",
       "      <td>https://webmasters.stackexchange.com/users/36029</td>\n",
       "      <td>http://webmasters.stackexchange.com/questions/...</td>\n",
       "      <td>TECHNOLOGY</td>\n",
       "      <td>webmasters.stackexchange.com</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      qa_id                                     question_title  \\\n",
       "2780   4429                       Dimension of a quotient ring   \n",
       "6013   9539  Is there a way to read the columns in a shapef...   \n",
       "2992   4770  Quantity based discount for single product in ...   \n",
       "150     238     hide javascript/jquery scripts from html page?   \n",
       "1827   2898        Which whois is the most frequently updated?   \n",
       "\n",
       "                                          question_body question_user_name  \\\n",
       "2780  \\n  What is the Krull dimension of $B=A[x,y,z]...               Zoey   \n",
       "6013  My shapefile has a number of fields, like road...            patrick   \n",
       "2992  I'm using latest build of Expresso Store, 2.3....           neekster   \n",
       "150   How do I hide my javascript/jquery scripts fro...   Shaitender Singh   \n",
       "1827  As the domain manager of 200+ domains, it's es...             Miss M   \n",
       "\n",
       "                                     question_user_page  \\\n",
       "2780        https://math.stackexchange.com/users/234074   \n",
       "6013            https://gis.stackexchange.com/users/733   \n",
       "2992  https://expressionengine.stackexchange.com/use...   \n",
       "150              https://stackoverflow.com/users/192252   \n",
       "1827   https://webmasters.stackexchange.com/users/52761   \n",
       "\n",
       "                                                 answer  \\\n",
       "2780  Your first ring $B$ is just $A[x^{\\pm1}]$, the...   \n",
       "6013  Yes, a shapefile in this context is a feature ...   \n",
       "2992  Use this bulk discounts add-on, does exactly w...   \n",
       "150   You can't, sorry. No matter what you do, even ...   \n",
       "1827  Being one who has a similar site, I would have...   \n",
       "\n",
       "            answer_user_name  \\\n",
       "2780  Mariano Suárez-Álvarez   \n",
       "6013              Petr Krebs   \n",
       "2992             Peter Lewis   \n",
       "150           Brian Campbell   \n",
       "1827               closetnoc   \n",
       "\n",
       "                                       answer_user_page  \\\n",
       "2780           https://math.stackexchange.com/users/274   \n",
       "6013            https://gis.stackexchange.com/users/434   \n",
       "2992  https://expressionengine.stackexchange.com/use...   \n",
       "150               https://stackoverflow.com/users/69755   \n",
       "1827   https://webmasters.stackexchange.com/users/36029   \n",
       "\n",
       "                                                    url       category  \\\n",
       "2780  http://math.stackexchange.com/questions/138610...        SCIENCE   \n",
       "6013  http://gis.stackexchange.com/questions/4619/is...     TECHNOLOGY   \n",
       "2992  http://expressionengine.stackexchange.com/ques...     TECHNOLOGY   \n",
       "150   http://stackoverflow.com/questions/1628799/hid...  STACKOVERFLOW   \n",
       "1827  http://webmasters.stackexchange.com/questions/...     TECHNOLOGY   \n",
       "\n",
       "                                    host  \n",
       "2780              math.stackexchange.com  \n",
       "6013               gis.stackexchange.com  \n",
       "2992  expressionengine.stackexchange.com  \n",
       "150                    stackoverflow.com  \n",
       "1827        webmasters.stackexchange.com  "
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_valid_x.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question_asker_intent_understanding</th>\n",
       "      <th>question_body_critical</th>\n",
       "      <th>question_conversational</th>\n",
       "      <th>question_expect_short_answer</th>\n",
       "      <th>question_fact_seeking</th>\n",
       "      <th>question_has_commonly_accepted_answer</th>\n",
       "      <th>question_interestingness_others</th>\n",
       "      <th>question_interestingness_self</th>\n",
       "      <th>question_multi_intent</th>\n",
       "      <th>question_not_really_a_question</th>\n",
       "      <th>...</th>\n",
       "      <th>question_well_written</th>\n",
       "      <th>answer_helpful</th>\n",
       "      <th>answer_level_of_information</th>\n",
       "      <th>answer_plausible</th>\n",
       "      <th>answer_relevance</th>\n",
       "      <th>answer_satisfaction</th>\n",
       "      <th>answer_type_instructions</th>\n",
       "      <th>answer_type_procedure</th>\n",
       "      <th>answer_type_reason_explanation</th>\n",
       "      <th>answer_well_written</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2780</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6013</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.888889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2992</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.888889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.866667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1827</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      question_asker_intent_understanding  question_body_critical  \\\n",
       "2780                             0.777778                0.444444   \n",
       "6013                             1.000000                1.000000   \n",
       "2992                             0.888889                0.444444   \n",
       "150                              1.000000                1.000000   \n",
       "1827                             0.777778                0.777778   \n",
       "\n",
       "      question_conversational  question_expect_short_answer  \\\n",
       "2780                 0.000000                      1.000000   \n",
       "6013                 0.000000                      0.666667   \n",
       "2992                 0.000000                      1.000000   \n",
       "150                  0.000000                      0.666667   \n",
       "1827                 0.666667                      1.000000   \n",
       "\n",
       "      question_fact_seeking  question_has_commonly_accepted_answer  \\\n",
       "2780               1.000000                               0.666667   \n",
       "6013               0.333333                               1.000000   \n",
       "2992               1.000000                               1.000000   \n",
       "150                1.000000                               0.666667   \n",
       "1827               1.000000                               1.000000   \n",
       "\n",
       "      question_interestingness_others  question_interestingness_self  \\\n",
       "2780                         0.555556                       0.333333   \n",
       "6013                         0.444444                       0.333333   \n",
       "2992                         0.777778                       0.444444   \n",
       "150                          0.555556                       0.444444   \n",
       "1827                         0.666667                       0.333333   \n",
       "\n",
       "      question_multi_intent  question_not_really_a_question  ...  \\\n",
       "2780               0.000000                             0.0  ...   \n",
       "6013               0.000000                             0.0  ...   \n",
       "2992               0.666667                             0.0  ...   \n",
       "150                0.000000                             0.0  ...   \n",
       "1827               0.000000                             0.0  ...   \n",
       "\n",
       "      question_well_written  answer_helpful  answer_level_of_information  \\\n",
       "2780               0.666667        1.000000                     0.777778   \n",
       "6013               0.888889        1.000000                     0.666667   \n",
       "2992               0.444444        1.000000                     0.666667   \n",
       "150                0.888889        1.000000                     0.777778   \n",
       "1827               1.000000        0.888889                     0.777778   \n",
       "\n",
       "      answer_plausible  answer_relevance  answer_satisfaction  \\\n",
       "2780          1.000000               1.0             0.933333   \n",
       "6013          1.000000               1.0             0.933333   \n",
       "2992          1.000000               1.0             1.000000   \n",
       "150           1.000000               1.0             0.866667   \n",
       "1827          0.888889               1.0             0.800000   \n",
       "\n",
       "      answer_type_instructions  answer_type_procedure  \\\n",
       "2780                  0.333333               0.000000   \n",
       "6013                  0.666667               0.000000   \n",
       "2992                  1.000000               0.000000   \n",
       "150                   1.000000               0.333333   \n",
       "1827                  0.000000               0.000000   \n",
       "\n",
       "      answer_type_reason_explanation  answer_well_written  \n",
       "2780                        0.333333             1.000000  \n",
       "6013                        0.333333             0.888889  \n",
       "2992                        0.000000             0.888889  \n",
       "150                         0.666667             1.000000  \n",
       "1827                        0.333333             1.000000  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_valid_y.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load trained models\n",
    "model_path = [f'bert/Full-0.h5', f'bert/bertuned_f0.h5', f'bert/bertuned_f1.h5', f'bert/bertuned_f2.h5', f'bert/bertuned_f3.h5', f'bert/bertuned_f4.h5']\n",
    "models = []\n",
    "\n",
    "for i in range(len(model_path)):\n",
    "    mp = model_path[i]\n",
    "    model = bert_model()\n",
    "    model.load_weights(mp)\n",
    "    models.append(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "476/476 [==============================] - 390s 818ms/sample\n"
     ]
    }
   ],
   "source": [
    "# Predict the labels of validation set\n",
    "valid_predictions = []\n",
    "\n",
    "for model in models[:1]:\n",
    "    valid_predictions.append(model.predict(valid_inputs, batch_size=8, verbose=1))\n",
    "\n",
    "final_predictions_valid = np.mean(valid_predictions, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "      <th>25</th>\n",
       "      <th>26</th>\n",
       "      <th>27</th>\n",
       "      <th>28</th>\n",
       "      <th>29</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.869381</td>\n",
       "      <td>0.491549</td>\n",
       "      <td>0.007663</td>\n",
       "      <td>0.987843</td>\n",
       "      <td>0.982960</td>\n",
       "      <td>0.594069</td>\n",
       "      <td>0.570143</td>\n",
       "      <td>0.319139</td>\n",
       "      <td>0.018902</td>\n",
       "      <td>0.001266</td>\n",
       "      <td>...</td>\n",
       "      <td>0.829597</td>\n",
       "      <td>0.991001</td>\n",
       "      <td>0.713615</td>\n",
       "      <td>0.990622</td>\n",
       "      <td>0.997140</td>\n",
       "      <td>0.957748</td>\n",
       "      <td>0.215083</td>\n",
       "      <td>0.009544</td>\n",
       "      <td>0.214697</td>\n",
       "      <td>0.967283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.987236</td>\n",
       "      <td>0.913054</td>\n",
       "      <td>0.002851</td>\n",
       "      <td>0.818194</td>\n",
       "      <td>0.262798</td>\n",
       "      <td>0.989817</td>\n",
       "      <td>0.405797</td>\n",
       "      <td>0.284093</td>\n",
       "      <td>0.006089</td>\n",
       "      <td>0.002051</td>\n",
       "      <td>...</td>\n",
       "      <td>0.939046</td>\n",
       "      <td>0.993420</td>\n",
       "      <td>0.702525</td>\n",
       "      <td>0.997489</td>\n",
       "      <td>0.997331</td>\n",
       "      <td>0.961701</td>\n",
       "      <td>0.451416</td>\n",
       "      <td>0.018958</td>\n",
       "      <td>0.317158</td>\n",
       "      <td>0.955285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.946362</td>\n",
       "      <td>0.559000</td>\n",
       "      <td>0.005701</td>\n",
       "      <td>0.153690</td>\n",
       "      <td>0.886456</td>\n",
       "      <td>0.964760</td>\n",
       "      <td>0.368762</td>\n",
       "      <td>0.363530</td>\n",
       "      <td>0.134047</td>\n",
       "      <td>0.005127</td>\n",
       "      <td>...</td>\n",
       "      <td>0.689516</td>\n",
       "      <td>0.986197</td>\n",
       "      <td>0.603693</td>\n",
       "      <td>0.992347</td>\n",
       "      <td>0.991689</td>\n",
       "      <td>0.804043</td>\n",
       "      <td>0.963588</td>\n",
       "      <td>0.076932</td>\n",
       "      <td>0.037348</td>\n",
       "      <td>0.901946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.977030</td>\n",
       "      <td>0.689247</td>\n",
       "      <td>0.004422</td>\n",
       "      <td>0.971632</td>\n",
       "      <td>0.898258</td>\n",
       "      <td>0.885676</td>\n",
       "      <td>0.627077</td>\n",
       "      <td>0.393279</td>\n",
       "      <td>0.008055</td>\n",
       "      <td>0.000502</td>\n",
       "      <td>...</td>\n",
       "      <td>0.783575</td>\n",
       "      <td>0.992845</td>\n",
       "      <td>0.656368</td>\n",
       "      <td>0.996941</td>\n",
       "      <td>0.996181</td>\n",
       "      <td>0.927622</td>\n",
       "      <td>0.816781</td>\n",
       "      <td>0.196169</td>\n",
       "      <td>0.906635</td>\n",
       "      <td>0.977425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.981712</td>\n",
       "      <td>0.708787</td>\n",
       "      <td>0.003829</td>\n",
       "      <td>0.979621</td>\n",
       "      <td>0.950203</td>\n",
       "      <td>0.153332</td>\n",
       "      <td>0.711191</td>\n",
       "      <td>0.507949</td>\n",
       "      <td>0.174303</td>\n",
       "      <td>0.000851</td>\n",
       "      <td>...</td>\n",
       "      <td>0.960060</td>\n",
       "      <td>0.970717</td>\n",
       "      <td>0.751022</td>\n",
       "      <td>0.978112</td>\n",
       "      <td>0.988915</td>\n",
       "      <td>0.865172</td>\n",
       "      <td>0.116016</td>\n",
       "      <td>0.242364</td>\n",
       "      <td>0.919507</td>\n",
       "      <td>0.956978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>471</td>\n",
       "      <td>0.857350</td>\n",
       "      <td>0.769211</td>\n",
       "      <td>0.012732</td>\n",
       "      <td>0.596025</td>\n",
       "      <td>0.964083</td>\n",
       "      <td>0.889377</td>\n",
       "      <td>0.578748</td>\n",
       "      <td>0.362019</td>\n",
       "      <td>0.384986</td>\n",
       "      <td>0.002616</td>\n",
       "      <td>...</td>\n",
       "      <td>0.914520</td>\n",
       "      <td>0.958980</td>\n",
       "      <td>0.584990</td>\n",
       "      <td>0.987119</td>\n",
       "      <td>0.988563</td>\n",
       "      <td>0.855754</td>\n",
       "      <td>0.287681</td>\n",
       "      <td>0.459318</td>\n",
       "      <td>0.971183</td>\n",
       "      <td>0.947662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>472</td>\n",
       "      <td>0.959904</td>\n",
       "      <td>0.536496</td>\n",
       "      <td>0.004854</td>\n",
       "      <td>0.860072</td>\n",
       "      <td>0.977445</td>\n",
       "      <td>0.960938</td>\n",
       "      <td>0.607823</td>\n",
       "      <td>0.564451</td>\n",
       "      <td>0.038689</td>\n",
       "      <td>0.001744</td>\n",
       "      <td>...</td>\n",
       "      <td>0.877827</td>\n",
       "      <td>0.943477</td>\n",
       "      <td>0.634282</td>\n",
       "      <td>0.980762</td>\n",
       "      <td>0.994144</td>\n",
       "      <td>0.878152</td>\n",
       "      <td>0.924284</td>\n",
       "      <td>0.350154</td>\n",
       "      <td>0.083588</td>\n",
       "      <td>0.895772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>473</td>\n",
       "      <td>0.953137</td>\n",
       "      <td>0.914440</td>\n",
       "      <td>0.005238</td>\n",
       "      <td>0.964677</td>\n",
       "      <td>0.976245</td>\n",
       "      <td>0.984673</td>\n",
       "      <td>0.712662</td>\n",
       "      <td>0.537255</td>\n",
       "      <td>0.004460</td>\n",
       "      <td>0.002185</td>\n",
       "      <td>...</td>\n",
       "      <td>0.920890</td>\n",
       "      <td>0.983903</td>\n",
       "      <td>0.749031</td>\n",
       "      <td>0.996012</td>\n",
       "      <td>0.997102</td>\n",
       "      <td>0.941693</td>\n",
       "      <td>0.579897</td>\n",
       "      <td>0.399180</td>\n",
       "      <td>0.419634</td>\n",
       "      <td>0.970456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>474</td>\n",
       "      <td>0.841125</td>\n",
       "      <td>0.376478</td>\n",
       "      <td>0.007780</td>\n",
       "      <td>0.990137</td>\n",
       "      <td>0.961846</td>\n",
       "      <td>0.880656</td>\n",
       "      <td>0.562385</td>\n",
       "      <td>0.458296</td>\n",
       "      <td>0.439309</td>\n",
       "      <td>0.006095</td>\n",
       "      <td>...</td>\n",
       "      <td>0.801846</td>\n",
       "      <td>0.976788</td>\n",
       "      <td>0.760943</td>\n",
       "      <td>0.973384</td>\n",
       "      <td>0.990753</td>\n",
       "      <td>0.902006</td>\n",
       "      <td>0.991708</td>\n",
       "      <td>0.008178</td>\n",
       "      <td>0.040411</td>\n",
       "      <td>0.943300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>475</td>\n",
       "      <td>0.856106</td>\n",
       "      <td>0.417290</td>\n",
       "      <td>0.005800</td>\n",
       "      <td>0.955756</td>\n",
       "      <td>0.965146</td>\n",
       "      <td>0.983868</td>\n",
       "      <td>0.658676</td>\n",
       "      <td>0.433161</td>\n",
       "      <td>0.982508</td>\n",
       "      <td>0.002302</td>\n",
       "      <td>...</td>\n",
       "      <td>0.903786</td>\n",
       "      <td>0.973717</td>\n",
       "      <td>0.692099</td>\n",
       "      <td>0.990930</td>\n",
       "      <td>0.995555</td>\n",
       "      <td>0.931709</td>\n",
       "      <td>0.336327</td>\n",
       "      <td>0.149081</td>\n",
       "      <td>0.975570</td>\n",
       "      <td>0.965319</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>476 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6   \\\n",
       "0    0.869381  0.491549  0.007663  0.987843  0.982960  0.594069  0.570143   \n",
       "1    0.987236  0.913054  0.002851  0.818194  0.262798  0.989817  0.405797   \n",
       "2    0.946362  0.559000  0.005701  0.153690  0.886456  0.964760  0.368762   \n",
       "3    0.977030  0.689247  0.004422  0.971632  0.898258  0.885676  0.627077   \n",
       "4    0.981712  0.708787  0.003829  0.979621  0.950203  0.153332  0.711191   \n",
       "..        ...       ...       ...       ...       ...       ...       ...   \n",
       "471  0.857350  0.769211  0.012732  0.596025  0.964083  0.889377  0.578748   \n",
       "472  0.959904  0.536496  0.004854  0.860072  0.977445  0.960938  0.607823   \n",
       "473  0.953137  0.914440  0.005238  0.964677  0.976245  0.984673  0.712662   \n",
       "474  0.841125  0.376478  0.007780  0.990137  0.961846  0.880656  0.562385   \n",
       "475  0.856106  0.417290  0.005800  0.955756  0.965146  0.983868  0.658676   \n",
       "\n",
       "           7         8         9   ...        20        21        22  \\\n",
       "0    0.319139  0.018902  0.001266  ...  0.829597  0.991001  0.713615   \n",
       "1    0.284093  0.006089  0.002051  ...  0.939046  0.993420  0.702525   \n",
       "2    0.363530  0.134047  0.005127  ...  0.689516  0.986197  0.603693   \n",
       "3    0.393279  0.008055  0.000502  ...  0.783575  0.992845  0.656368   \n",
       "4    0.507949  0.174303  0.000851  ...  0.960060  0.970717  0.751022   \n",
       "..        ...       ...       ...  ...       ...       ...       ...   \n",
       "471  0.362019  0.384986  0.002616  ...  0.914520  0.958980  0.584990   \n",
       "472  0.564451  0.038689  0.001744  ...  0.877827  0.943477  0.634282   \n",
       "473  0.537255  0.004460  0.002185  ...  0.920890  0.983903  0.749031   \n",
       "474  0.458296  0.439309  0.006095  ...  0.801846  0.976788  0.760943   \n",
       "475  0.433161  0.982508  0.002302  ...  0.903786  0.973717  0.692099   \n",
       "\n",
       "           23        24        25        26        27        28        29  \n",
       "0    0.990622  0.997140  0.957748  0.215083  0.009544  0.214697  0.967283  \n",
       "1    0.997489  0.997331  0.961701  0.451416  0.018958  0.317158  0.955285  \n",
       "2    0.992347  0.991689  0.804043  0.963588  0.076932  0.037348  0.901946  \n",
       "3    0.996941  0.996181  0.927622  0.816781  0.196169  0.906635  0.977425  \n",
       "4    0.978112  0.988915  0.865172  0.116016  0.242364  0.919507  0.956978  \n",
       "..        ...       ...       ...       ...       ...       ...       ...  \n",
       "471  0.987119  0.988563  0.855754  0.287681  0.459318  0.971183  0.947662  \n",
       "472  0.980762  0.994144  0.878152  0.924284  0.350154  0.083588  0.895772  \n",
       "473  0.996012  0.997102  0.941693  0.579897  0.399180  0.419634  0.970456  \n",
       "474  0.973384  0.990753  0.902006  0.991708  0.008178  0.040411  0.943300  \n",
       "475  0.990930  0.995555  0.931709  0.336327  0.149081  0.975570  0.965319  \n",
       "\n",
       "[476 rows x 30 columns]"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_valid_pred = pd.DataFrame(final_predictions_valid)\n",
    "df_valid_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question_asker_intent_understanding</th>\n",
       "      <th>question_body_critical</th>\n",
       "      <th>question_conversational</th>\n",
       "      <th>question_expect_short_answer</th>\n",
       "      <th>question_fact_seeking</th>\n",
       "      <th>question_has_commonly_accepted_answer</th>\n",
       "      <th>question_interestingness_others</th>\n",
       "      <th>question_interestingness_self</th>\n",
       "      <th>question_multi_intent</th>\n",
       "      <th>question_not_really_a_question</th>\n",
       "      <th>...</th>\n",
       "      <th>question_well_written</th>\n",
       "      <th>answer_helpful</th>\n",
       "      <th>answer_level_of_information</th>\n",
       "      <th>answer_plausible</th>\n",
       "      <th>answer_relevance</th>\n",
       "      <th>answer_satisfaction</th>\n",
       "      <th>answer_type_instructions</th>\n",
       "      <th>answer_type_procedure</th>\n",
       "      <th>answer_type_reason_explanation</th>\n",
       "      <th>answer_well_written</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2780</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6013</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.888889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2992</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.888889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.866667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1827</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3563</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.888889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1999</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.777778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>213</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4232</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.888889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2597</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.888889</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>476 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      question_asker_intent_understanding  question_body_critical  \\\n",
       "2780                             0.777778                0.444444   \n",
       "6013                             1.000000                1.000000   \n",
       "2992                             0.888889                0.444444   \n",
       "150                              1.000000                1.000000   \n",
       "1827                             0.777778                0.777778   \n",
       "...                                   ...                     ...   \n",
       "3563                             0.666667                0.777778   \n",
       "1999                             0.777778                0.555556   \n",
       "213                              0.888889                0.888889   \n",
       "4232                             0.777778                0.333333   \n",
       "2597                             1.000000                0.333333   \n",
       "\n",
       "      question_conversational  question_expect_short_answer  \\\n",
       "2780                 0.000000                      1.000000   \n",
       "6013                 0.000000                      0.666667   \n",
       "2992                 0.000000                      1.000000   \n",
       "150                  0.000000                      0.666667   \n",
       "1827                 0.666667                      1.000000   \n",
       "...                       ...                           ...   \n",
       "3563                 0.000000                      0.333333   \n",
       "1999                 0.000000                      0.333333   \n",
       "213                  0.000000                      1.000000   \n",
       "4232                 0.000000                      1.000000   \n",
       "2597                 0.000000                      0.666667   \n",
       "\n",
       "      question_fact_seeking  question_has_commonly_accepted_answer  \\\n",
       "2780               1.000000                               0.666667   \n",
       "6013               0.333333                               1.000000   \n",
       "2992               1.000000                               1.000000   \n",
       "150                1.000000                               0.666667   \n",
       "1827               1.000000                               1.000000   \n",
       "...                     ...                                    ...   \n",
       "3563               1.000000                               0.666667   \n",
       "1999               1.000000                               0.666667   \n",
       "213                1.000000                               1.000000   \n",
       "4232               1.000000                               0.666667   \n",
       "2597               1.000000                               1.000000   \n",
       "\n",
       "      question_interestingness_others  question_interestingness_self  \\\n",
       "2780                         0.555556                       0.333333   \n",
       "6013                         0.444444                       0.333333   \n",
       "2992                         0.777778                       0.444444   \n",
       "150                          0.555556                       0.444444   \n",
       "1827                         0.666667                       0.333333   \n",
       "...                               ...                            ...   \n",
       "3563                         0.666667                       0.444444   \n",
       "1999                         0.666667                       0.444444   \n",
       "213                          0.666667                       0.555556   \n",
       "4232                         0.333333                       0.555556   \n",
       "2597                         0.444444                       0.333333   \n",
       "\n",
       "      question_multi_intent  question_not_really_a_question  ...  \\\n",
       "2780               0.000000                             0.0  ...   \n",
       "6013               0.000000                             0.0  ...   \n",
       "2992               0.666667                             0.0  ...   \n",
       "150                0.000000                             0.0  ...   \n",
       "1827               0.000000                             0.0  ...   \n",
       "...                     ...                             ...  ...   \n",
       "3563               0.333333                             0.0  ...   \n",
       "1999               0.333333                             0.0  ...   \n",
       "213                0.000000                             0.0  ...   \n",
       "4232               0.333333                             0.0  ...   \n",
       "2597               1.000000                             0.0  ...   \n",
       "\n",
       "      question_well_written  answer_helpful  answer_level_of_information  \\\n",
       "2780               0.666667        1.000000                     0.777778   \n",
       "6013               0.888889        1.000000                     0.666667   \n",
       "2992               0.444444        1.000000                     0.666667   \n",
       "150                0.888889        1.000000                     0.777778   \n",
       "1827               1.000000        0.888889                     0.777778   \n",
       "...                     ...             ...                          ...   \n",
       "3563               0.777778        1.000000                     0.666667   \n",
       "1999               0.666667        0.777778                     0.500000   \n",
       "213                0.888889        1.000000                     0.666667   \n",
       "4232               0.777778        1.000000                     0.777778   \n",
       "2597               0.555556        1.000000                     0.666667   \n",
       "\n",
       "      answer_plausible  answer_relevance  answer_satisfaction  \\\n",
       "2780          1.000000          1.000000             0.933333   \n",
       "6013          1.000000          1.000000             0.933333   \n",
       "2992          1.000000          1.000000             1.000000   \n",
       "150           1.000000          1.000000             0.866667   \n",
       "1827          0.888889          1.000000             0.800000   \n",
       "...                ...               ...                  ...   \n",
       "3563          1.000000          1.000000             0.800000   \n",
       "1999          0.888889          1.000000             0.666667   \n",
       "213           1.000000          1.000000             0.933333   \n",
       "4232          1.000000          0.833333             0.800000   \n",
       "2597          1.000000          1.000000             0.933333   \n",
       "\n",
       "      answer_type_instructions  answer_type_procedure  \\\n",
       "2780                  0.333333               0.000000   \n",
       "6013                  0.666667               0.000000   \n",
       "2992                  1.000000               0.000000   \n",
       "150                   1.000000               0.333333   \n",
       "1827                  0.000000               0.000000   \n",
       "...                        ...                    ...   \n",
       "3563                  0.333333               0.333333   \n",
       "1999                  0.333333               0.333333   \n",
       "213                   0.666667               0.333333   \n",
       "4232                  1.000000               0.000000   \n",
       "2597                  0.000000               0.000000   \n",
       "\n",
       "      answer_type_reason_explanation  answer_well_written  \n",
       "2780                        0.333333             1.000000  \n",
       "6013                        0.333333             0.888889  \n",
       "2992                        0.000000             0.888889  \n",
       "150                         0.666667             1.000000  \n",
       "1827                        0.333333             1.000000  \n",
       "...                              ...                  ...  \n",
       "3563                        1.000000             0.888889  \n",
       "1999                        0.666667             0.777778  \n",
       "213                         0.333333             1.000000  \n",
       "4232                        0.000000             0.888889  \n",
       "2597                        1.000000             0.888889  \n",
       "\n",
       "[476 rows x 30 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_valid_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qa_id</th>\n",
       "      <th>question_title</th>\n",
       "      <th>question_body</th>\n",
       "      <th>question_user_name</th>\n",
       "      <th>question_user_page</th>\n",
       "      <th>answer</th>\n",
       "      <th>answer_user_name</th>\n",
       "      <th>answer_user_page</th>\n",
       "      <th>url</th>\n",
       "      <th>category</th>\n",
       "      <th>host</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>39</td>\n",
       "      <td>Will leaving corpses lying around upset my pri...</td>\n",
       "      <td>I see questions/information online about how t...</td>\n",
       "      <td>Dylan</td>\n",
       "      <td>https://gaming.stackexchange.com/users/64471</td>\n",
       "      <td>There is no consequence for leaving corpses an...</td>\n",
       "      <td>Nelson868</td>\n",
       "      <td>https://gaming.stackexchange.com/users/97324</td>\n",
       "      <td>http://gaming.stackexchange.com/questions/1979...</td>\n",
       "      <td>CULTURE</td>\n",
       "      <td>gaming.stackexchange.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>46</td>\n",
       "      <td>Url link to feature image in the portfolio</td>\n",
       "      <td>I am new to Wordpress. i have issue with Featu...</td>\n",
       "      <td>Anu</td>\n",
       "      <td>https://wordpress.stackexchange.com/users/72927</td>\n",
       "      <td>I think it is possible with custom fields.\\n\\n...</td>\n",
       "      <td>Irina</td>\n",
       "      <td>https://wordpress.stackexchange.com/users/27233</td>\n",
       "      <td>http://wordpress.stackexchange.com/questions/1...</td>\n",
       "      <td>TECHNOLOGY</td>\n",
       "      <td>wordpress.stackexchange.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>70</td>\n",
       "      <td>Is accuracy, recoil or bullet spread affected ...</td>\n",
       "      <td>To experiment I started a bot game, toggled in...</td>\n",
       "      <td>Konsta</td>\n",
       "      <td>https://gaming.stackexchange.com/users/37545</td>\n",
       "      <td>You do not have armour in the screenshots. Thi...</td>\n",
       "      <td>Damon Smithies</td>\n",
       "      <td>https://gaming.stackexchange.com/users/70641</td>\n",
       "      <td>http://gaming.stackexchange.com/questions/2154...</td>\n",
       "      <td>CULTURE</td>\n",
       "      <td>gaming.stackexchange.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>132</td>\n",
       "      <td>Suddenly got an I/O error from my external HDD</td>\n",
       "      <td>I have used my Raspberry Pi as a torrent-serve...</td>\n",
       "      <td>robbannn</td>\n",
       "      <td>https://raspberrypi.stackexchange.com/users/17341</td>\n",
       "      <td>Your Western Digital hard drive is disappearin...</td>\n",
       "      <td>HeatfanJohn</td>\n",
       "      <td>https://raspberrypi.stackexchange.com/users/1311</td>\n",
       "      <td>http://raspberrypi.stackexchange.com/questions...</td>\n",
       "      <td>TECHNOLOGY</td>\n",
       "      <td>raspberrypi.stackexchange.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>200</td>\n",
       "      <td>Passenger Name - Flight Booking Passenger only...</td>\n",
       "      <td>I have bought Delhi-London return flights for ...</td>\n",
       "      <td>Amit</td>\n",
       "      <td>https://travel.stackexchange.com/users/29089</td>\n",
       "      <td>I called two persons who work for Saudia (tick...</td>\n",
       "      <td>Nean Der Thal</td>\n",
       "      <td>https://travel.stackexchange.com/users/10051</td>\n",
       "      <td>http://travel.stackexchange.com/questions/4704...</td>\n",
       "      <td>CULTURE</td>\n",
       "      <td>travel.stackexchange.com</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   qa_id                                     question_title  \\\n",
       "0     39  Will leaving corpses lying around upset my pri...   \n",
       "1     46         Url link to feature image in the portfolio   \n",
       "2     70  Is accuracy, recoil or bullet spread affected ...   \n",
       "3    132     Suddenly got an I/O error from my external HDD   \n",
       "4    200  Passenger Name - Flight Booking Passenger only...   \n",
       "\n",
       "                                       question_body question_user_name  \\\n",
       "0  I see questions/information online about how t...              Dylan   \n",
       "1  I am new to Wordpress. i have issue with Featu...                Anu   \n",
       "2  To experiment I started a bot game, toggled in...             Konsta   \n",
       "3  I have used my Raspberry Pi as a torrent-serve...           robbannn   \n",
       "4  I have bought Delhi-London return flights for ...               Amit   \n",
       "\n",
       "                                  question_user_page  \\\n",
       "0       https://gaming.stackexchange.com/users/64471   \n",
       "1    https://wordpress.stackexchange.com/users/72927   \n",
       "2       https://gaming.stackexchange.com/users/37545   \n",
       "3  https://raspberrypi.stackexchange.com/users/17341   \n",
       "4       https://travel.stackexchange.com/users/29089   \n",
       "\n",
       "                                              answer answer_user_name  \\\n",
       "0  There is no consequence for leaving corpses an...        Nelson868   \n",
       "1  I think it is possible with custom fields.\\n\\n...            Irina   \n",
       "2  You do not have armour in the screenshots. Thi...   Damon Smithies   \n",
       "3  Your Western Digital hard drive is disappearin...      HeatfanJohn   \n",
       "4  I called two persons who work for Saudia (tick...    Nean Der Thal   \n",
       "\n",
       "                                   answer_user_page  \\\n",
       "0      https://gaming.stackexchange.com/users/97324   \n",
       "1   https://wordpress.stackexchange.com/users/27233   \n",
       "2      https://gaming.stackexchange.com/users/70641   \n",
       "3  https://raspberrypi.stackexchange.com/users/1311   \n",
       "4      https://travel.stackexchange.com/users/10051   \n",
       "\n",
       "                                                 url    category  \\\n",
       "0  http://gaming.stackexchange.com/questions/1979...     CULTURE   \n",
       "1  http://wordpress.stackexchange.com/questions/1...  TECHNOLOGY   \n",
       "2  http://gaming.stackexchange.com/questions/2154...     CULTURE   \n",
       "3  http://raspberrypi.stackexchange.com/questions...  TECHNOLOGY   \n",
       "4  http://travel.stackexchange.com/questions/4704...     CULTURE   \n",
       "\n",
       "                            host  \n",
       "0       gaming.stackexchange.com  \n",
       "1    wordpress.stackexchange.com  \n",
       "2       gaming.stackexchange.com  \n",
       "3  raspberrypi.stackexchange.com  \n",
       "4       travel.stackexchange.com  "
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check the scores using spearmanr from scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "      <th>Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>question_asker_intent_understanding</td>\n",
       "      <td>0.737226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>question_body_critical</td>\n",
       "      <td>0.881443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>question_conversational</td>\n",
       "      <td>0.512731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>question_expect_short_answer</td>\n",
       "      <td>0.813718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>question_fact_seeking</td>\n",
       "      <td>0.833274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>question_has_commonly_accepted_answer</td>\n",
       "      <td>0.776549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>question_interestingness_others</td>\n",
       "      <td>0.714254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>question_interestingness_self</td>\n",
       "      <td>0.809268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>question_multi_intent</td>\n",
       "      <td>0.823193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>question_not_really_a_question</td>\n",
       "      <td>0.157999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>question_opinion_seeking</td>\n",
       "      <td>0.870634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>question_type_choice</td>\n",
       "      <td>0.863672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>question_type_compare</td>\n",
       "      <td>0.408054</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>question_type_consequence</td>\n",
       "      <td>0.187766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>question_type_definition</td>\n",
       "      <td>0.397206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>question_type_entity</td>\n",
       "      <td>0.500752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>question_type_instructions</td>\n",
       "      <td>0.908582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>question_type_procedure</td>\n",
       "      <td>0.788041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>question_type_reason_explanation</td>\n",
       "      <td>0.894204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>question_type_spelling</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>question_well_written</td>\n",
       "      <td>0.822434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>answer_helpful</td>\n",
       "      <td>0.709452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>answer_level_of_information</td>\n",
       "      <td>0.640300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>answer_plausible</td>\n",
       "      <td>0.607229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>answer_relevance</td>\n",
       "      <td>0.481675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>answer_satisfaction</td>\n",
       "      <td>0.751804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>answer_type_instructions</td>\n",
       "      <td>0.913905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>answer_type_procedure</td>\n",
       "      <td>0.737042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>answer_type_reason_explanation</td>\n",
       "      <td>0.927172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>answer_well_written</td>\n",
       "      <td>0.741200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Label     Score\n",
       "0     question_asker_intent_understanding  0.737226\n",
       "1                  question_body_critical  0.881443\n",
       "2                 question_conversational  0.512731\n",
       "3            question_expect_short_answer  0.813718\n",
       "4                   question_fact_seeking  0.833274\n",
       "5   question_has_commonly_accepted_answer  0.776549\n",
       "6         question_interestingness_others  0.714254\n",
       "7           question_interestingness_self  0.809268\n",
       "8                   question_multi_intent  0.823193\n",
       "9          question_not_really_a_question  0.157999\n",
       "10               question_opinion_seeking  0.870634\n",
       "11                   question_type_choice  0.863672\n",
       "12                  question_type_compare  0.408054\n",
       "13              question_type_consequence  0.187766\n",
       "14               question_type_definition  0.397206\n",
       "15                   question_type_entity  0.500752\n",
       "16             question_type_instructions  0.908582\n",
       "17                question_type_procedure  0.788041\n",
       "18       question_type_reason_explanation  0.894204\n",
       "19                 question_type_spelling       NaN\n",
       "20                  question_well_written  0.822434\n",
       "21                         answer_helpful  0.709452\n",
       "22            answer_level_of_information  0.640300\n",
       "23                       answer_plausible  0.607229\n",
       "24                       answer_relevance  0.481675\n",
       "25                    answer_satisfaction  0.751804\n",
       "26               answer_type_instructions  0.913905\n",
       "27                  answer_type_procedure  0.737042\n",
       "28         answer_type_reason_explanation  0.927172\n",
       "29                    answer_well_written  0.741200"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# performance of each column in the validation dataset, model 1\n",
    "from scipy.stats import spearmanr\n",
    "\n",
    "df_spearman = pd.DataFrame(df_valid_y.columns,columns=[\"Label\"])\n",
    "coefs = []\n",
    "\n",
    "for i in range(30):\n",
    "    coef,p = spearmanr(df_valid_pred.iloc[:,i], df_valid_y.iloc[:,i])\n",
    "    coefs.append(coef)\n",
    "\n",
    "df_spearman[\"Score\"] = coefs\n",
    "df_spearman"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The scores of question_not_really_a_question(Label 10, 0.157999), question_type_consequence(Label 14, 0.187766) and question_type_spelling(Label 20, NaN) is significantly lower than others."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After checking the train set, we find the data of these columns is imbalanced. So, we're considering solving this problem using imblearn or other methods."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run the model on the test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_predictions = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "for model in models:\n",
    "    test_predictions.append(model.predict(test_inputs, batch_size=8, verbose=1)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_predictions = np.mean(test_predictions, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sub.iloc[:, 1:] = final_predictions\n",
    "df_sub.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qa_id</th>\n",
       "      <th>question_asker_intent_understanding</th>\n",
       "      <th>question_body_critical</th>\n",
       "      <th>question_conversational</th>\n",
       "      <th>question_expect_short_answer</th>\n",
       "      <th>question_fact_seeking</th>\n",
       "      <th>question_has_commonly_accepted_answer</th>\n",
       "      <th>question_interestingness_others</th>\n",
       "      <th>question_interestingness_self</th>\n",
       "      <th>question_multi_intent</th>\n",
       "      <th>...</th>\n",
       "      <th>question_well_written</th>\n",
       "      <th>answer_helpful</th>\n",
       "      <th>answer_level_of_information</th>\n",
       "      <th>answer_plausible</th>\n",
       "      <th>answer_relevance</th>\n",
       "      <th>answer_satisfaction</th>\n",
       "      <th>answer_type_instructions</th>\n",
       "      <th>answer_type_procedure</th>\n",
       "      <th>answer_type_reason_explanation</th>\n",
       "      <th>answer_well_written</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>39</td>\n",
       "      <td>0.938958</td>\n",
       "      <td>0.627481</td>\n",
       "      <td>0.166800</td>\n",
       "      <td>0.514917</td>\n",
       "      <td>0.576923</td>\n",
       "      <td>0.622745</td>\n",
       "      <td>0.676879</td>\n",
       "      <td>0.648084</td>\n",
       "      <td>0.729218</td>\n",
       "      <td>...</td>\n",
       "      <td>0.887719</td>\n",
       "      <td>0.919156</td>\n",
       "      <td>0.604636</td>\n",
       "      <td>0.973368</td>\n",
       "      <td>0.963013</td>\n",
       "      <td>0.798283</td>\n",
       "      <td>0.028482</td>\n",
       "      <td>0.020496</td>\n",
       "      <td>0.898880</td>\n",
       "      <td>0.909044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>46</td>\n",
       "      <td>0.881514</td>\n",
       "      <td>0.450305</td>\n",
       "      <td>0.006302</td>\n",
       "      <td>0.778764</td>\n",
       "      <td>0.729956</td>\n",
       "      <td>0.920597</td>\n",
       "      <td>0.584292</td>\n",
       "      <td>0.499064</td>\n",
       "      <td>0.071643</td>\n",
       "      <td>...</td>\n",
       "      <td>0.669986</td>\n",
       "      <td>0.962445</td>\n",
       "      <td>0.635315</td>\n",
       "      <td>0.980587</td>\n",
       "      <td>0.979045</td>\n",
       "      <td>0.878402</td>\n",
       "      <td>0.947890</td>\n",
       "      <td>0.075705</td>\n",
       "      <td>0.032106</td>\n",
       "      <td>0.904202</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>70</td>\n",
       "      <td>0.884626</td>\n",
       "      <td>0.625738</td>\n",
       "      <td>0.013018</td>\n",
       "      <td>0.840316</td>\n",
       "      <td>0.880019</td>\n",
       "      <td>0.968759</td>\n",
       "      <td>0.609891</td>\n",
       "      <td>0.463821</td>\n",
       "      <td>0.160505</td>\n",
       "      <td>...</td>\n",
       "      <td>0.830779</td>\n",
       "      <td>0.925678</td>\n",
       "      <td>0.587583</td>\n",
       "      <td>0.966551</td>\n",
       "      <td>0.969164</td>\n",
       "      <td>0.795232</td>\n",
       "      <td>0.072927</td>\n",
       "      <td>0.044401</td>\n",
       "      <td>0.845021</td>\n",
       "      <td>0.905604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>132</td>\n",
       "      <td>0.890446</td>\n",
       "      <td>0.398783</td>\n",
       "      <td>0.005364</td>\n",
       "      <td>0.725932</td>\n",
       "      <td>0.784621</td>\n",
       "      <td>0.939685</td>\n",
       "      <td>0.550969</td>\n",
       "      <td>0.413887</td>\n",
       "      <td>0.067337</td>\n",
       "      <td>...</td>\n",
       "      <td>0.683051</td>\n",
       "      <td>0.943653</td>\n",
       "      <td>0.678593</td>\n",
       "      <td>0.972033</td>\n",
       "      <td>0.981260</td>\n",
       "      <td>0.889558</td>\n",
       "      <td>0.818465</td>\n",
       "      <td>0.123610</td>\n",
       "      <td>0.496863</td>\n",
       "      <td>0.899144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>200</td>\n",
       "      <td>0.916232</td>\n",
       "      <td>0.408290</td>\n",
       "      <td>0.021440</td>\n",
       "      <td>0.886373</td>\n",
       "      <td>0.700372</td>\n",
       "      <td>0.888922</td>\n",
       "      <td>0.655180</td>\n",
       "      <td>0.603831</td>\n",
       "      <td>0.033444</td>\n",
       "      <td>...</td>\n",
       "      <td>0.631891</td>\n",
       "      <td>0.926608</td>\n",
       "      <td>0.628834</td>\n",
       "      <td>0.975340</td>\n",
       "      <td>0.963328</td>\n",
       "      <td>0.849761</td>\n",
       "      <td>0.226559</td>\n",
       "      <td>0.118528</td>\n",
       "      <td>0.615895</td>\n",
       "      <td>0.899786</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   qa_id  question_asker_intent_understanding  question_body_critical  \\\n",
       "0     39                             0.938958                0.627481   \n",
       "1     46                             0.881514                0.450305   \n",
       "2     70                             0.884626                0.625738   \n",
       "3    132                             0.890446                0.398783   \n",
       "4    200                             0.916232                0.408290   \n",
       "\n",
       "   question_conversational  question_expect_short_answer  \\\n",
       "0                 0.166800                      0.514917   \n",
       "1                 0.006302                      0.778764   \n",
       "2                 0.013018                      0.840316   \n",
       "3                 0.005364                      0.725932   \n",
       "4                 0.021440                      0.886373   \n",
       "\n",
       "   question_fact_seeking  question_has_commonly_accepted_answer  \\\n",
       "0               0.576923                               0.622745   \n",
       "1               0.729956                               0.920597   \n",
       "2               0.880019                               0.968759   \n",
       "3               0.784621                               0.939685   \n",
       "4               0.700372                               0.888922   \n",
       "\n",
       "   question_interestingness_others  question_interestingness_self  \\\n",
       "0                         0.676879                       0.648084   \n",
       "1                         0.584292                       0.499064   \n",
       "2                         0.609891                       0.463821   \n",
       "3                         0.550969                       0.413887   \n",
       "4                         0.655180                       0.603831   \n",
       "\n",
       "   question_multi_intent  ...  question_well_written  answer_helpful  \\\n",
       "0               0.729218  ...               0.887719        0.919156   \n",
       "1               0.071643  ...               0.669986        0.962445   \n",
       "2               0.160505  ...               0.830779        0.925678   \n",
       "3               0.067337  ...               0.683051        0.943653   \n",
       "4               0.033444  ...               0.631891        0.926608   \n",
       "\n",
       "   answer_level_of_information  answer_plausible  answer_relevance  \\\n",
       "0                     0.604636          0.973368          0.963013   \n",
       "1                     0.635315          0.980587          0.979045   \n",
       "2                     0.587583          0.966551          0.969164   \n",
       "3                     0.678593          0.972033          0.981260   \n",
       "4                     0.628834          0.975340          0.963328   \n",
       "\n",
       "   answer_satisfaction  answer_type_instructions  answer_type_procedure  \\\n",
       "0             0.798283                  0.028482               0.020496   \n",
       "1             0.878402                  0.947890               0.075705   \n",
       "2             0.795232                  0.072927               0.044401   \n",
       "3             0.889558                  0.818465               0.123610   \n",
       "4             0.849761                  0.226559               0.118528   \n",
       "\n",
       "   answer_type_reason_explanation  answer_well_written  \n",
       "0                        0.898880             0.909044  \n",
       "1                        0.032106             0.904202  \n",
       "2                        0.845021             0.905604  \n",
       "3                        0.496863             0.899144  \n",
       "4                        0.615895             0.899786  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sub.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}