Stage 1.5 Naive-classifier-with-hand-crafted-featurs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sample_submission_stage_1.csv', 'test_stage_1.tsv', 'trees']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"data\"))\n",
    "import zipfile\n",
    "import sys\n",
    "import time\n",
    "\n",
    "from keras import backend, models, layers, initializers, regularizers, constraints, optimizers\n",
    "from keras import callbacks as kc\n",
    "from keras import optimizers as ko\n",
    "\n",
    "from sklearn.model_selection import cross_val_score, KFold, train_test_split\n",
    "from sklearn.metrics import log_loss\n",
    "import time\n",
    "\n",
    "\n",
    "dense_layer_sizes = [37]\n",
    "dropout_rate = 0.6\n",
    "learning_rate = 0.001\n",
    "n_fold = 5\n",
    "batch_size = 32\n",
    "epochs = 1000\n",
    "patience = 100\n",
    "# n_test = 100\n",
    "lambd = 0.1 # L2 regularization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The features are from this [awesome kernel](https://www.kaggle.com/pheell/look-ma-no-embeddings)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import spacy\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "nlp = spacy.load('en_core_web_lg')\n",
    "\n",
    "# Load the GAP data\n",
    "\n",
    "data = pd.concat([pd.read_csv('gap-development.tsv', sep='\\t'),\n",
    "                  pd.read_csv('gap-validation.tsv', sep='\\t'),\n",
    "                  pd.read_csv('gap-test.tsv', sep='\\t')\n",
    "                 ], ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def domain(t):\n",
    "    while not t._.subj and not t._.poss and\\\n",
    "            not (t.dep_ == 'xcomp' and t.head._.obj) and\\\n",
    "            t != t.head:\n",
    "        t = t.head\n",
    "    return t\n",
    "\n",
    "def ccom(t):\n",
    "    return [t2 for t2 in t.head._.d]\n",
    "\n",
    "# spacy extensions:\n",
    "#   doc._.to(offset) => t at text char offset\n",
    "#   t._.c => t's children (list)                     #   t._.d => t's descendents (list)\n",
    "#   t._.subj => t's subject else False               #   t._.obj => t's object else False\n",
    "#   t._.domain => t's syntactic domain               #   t._.ccom => t's c-command domain\n",
    "\n",
    "spacy.tokens.doc.Doc.set_extension(\n",
    "    'to', method=lambda doc, offset: [t for t in doc if t.idx == offset][0], force=True)\n",
    "spacy.tokens.token.Token.set_extension(\n",
    "    'c', getter=lambda t: [c for c in t.children], force=True)\n",
    "spacy.tokens.token.Token.set_extension(\n",
    "    'd', getter=lambda t: [c for c in t.sent if t in list(c.ancestors)], force=True)\n",
    "spacy.tokens.token.Token.set_extension(\n",
    "    'subj', getter=lambda t: ([c for c in t._.c if c.dep_.startswith('nsubj')] + [False])[0], force=True)\n",
    "spacy.tokens.token.Token.set_extension(\n",
    "    'obj', getter=lambda t: ([c for c in t._.c if c.dep_.startswith('dobj')] + [False])[0], force=True)\n",
    "spacy.tokens.token.Token.set_extension(\n",
    "    'poss', getter=lambda t: ([c for c in t._.c if c.dep_.startswith('poss')] + [False])[0], force=True)\n",
    "spacy.tokens.token.Token.set_extension(\n",
    "    'span', method=lambda t, t2: t.doc[t.i:t2.i] if t.i < t2.i else t.doc[t2.i:t.i], force=True)\n",
    "spacy.tokens.token.Token.set_extension('domain', getter=domain, force=True)\n",
    "spacy.tokens.token.Token.set_extension('ccom', getter=ccom, force=True)\n",
    "\n",
    "def applyDisq(condition, candidates, candidate_dict, debug = False):\n",
    "    badnames = sum([nameset(c, candidate_dict) for c in candidates if c in condition[0]], [])\n",
    "    badcands = [c for c in candidates if c.text in badnames]\n",
    "    if debug and len(badcands) > 0: print('Disqualified:', badcands, '<', condition[1])\n",
    "    return [c for c in candidates if c not in badcands]\n",
    "\n",
    "# Apply a list of disqualifying conditions\n",
    "def applyDisqs(conditions, candidates, candidate_dict, debug = False):\n",
    "    for condition in conditions:\n",
    "        if len(candidates) < 1: return candidates\n",
    "        candidates = applyDisq(condition, candidates, candidate_dict, debug)\n",
    "    return candidates\n",
    "\n",
    "# Pass the list of disqualifying conditions for possessive pronouns (his, her)\n",
    "def disqGen(t, candidates, candidate_dict, debug = False):\n",
    "    conds = [(t._.ccom,\n",
    "             \"disqualify candidates c-commanded by genpn; e.g. e.g. *Julia read his_i book about John_i's life.\"),\n",
    "             ([t2 for t2 in candidates if t in t2._.ccom and t2.head.dep_ == 'appos'],\n",
    "             \"disqualify candidates modified by an appositive with genpn; e.g. *I wanted to see John_i, his_i father.\")\n",
    "            ]\n",
    "    return applyDisqs(conds, candidates, candidate_dict, debug)\n",
    "\n",
    "# Pass the list of list of disqualifying conditions for other pronouns\n",
    "def disqOthers(t, candidates, candidate_dict, debug = False):\n",
    "    conds = [([t2 for t2 in t._.ccom if t2.i > t.i],\n",
    "             \"disqualify candidates c-commanded by pn, unless they were preposed;\\\n",
    "             e.g. *He_i cried before John_i laughed. vs. Before John_i laughed, he_i cried.\"),\n",
    "             ([t2 for t2 in candidates if t in t2._.ccom and t2._.domain == t._.domain\n",
    "              and not (t.head.text == 'with' and t.head.head.lemma_ == 'take')],\n",
    "             \"disqualify candidates that c-command pn, unless in different domain;\\\n",
    "             e.g. Mary said that *John_i hit him_i. vs. John_i said that Mary hit him_i;\\\n",
    "             random hard-coded exception: `take with'\"),\n",
    "             ([t2 for t2 in candidates if t2._.domain.dep_ == 'xcomp' and t2._.domain.head._.obj and t2 == t2._.domain.head._.obj],\n",
    "             \"for xcomps with subjects parsed as upstairs dobj, disallow coref with that dobj;\\\n",
    "             e.g. *Mary wanted John_i to forgive him_i.\")\n",
    "            ]\n",
    "    return applyDisqs(conds, candidates, candidate_dict, debug)\n",
    "\n",
    "# Decide whether possessive or not and call appropriate function\n",
    "def disq(t, candidates, candidate_dict, debug = False):\n",
    "    func = disqGen if t.dep_ == 'poss' else disqOthers\n",
    "    candidates = func(t, candidates, candidate_dict, debug)\n",
    "    return candidates\n",
    "\n",
    "def find_head(w, wo, doc):\n",
    "    t = False; backtrack = 0\n",
    "    while not t:\n",
    "        try:\n",
    "            t = doc._.to(wo)\n",
    "        except IndexError:\n",
    "            wo -= 1; backtrack += 1\n",
    "    while t.dep_ == 'compound' and t.head.idx >= wo and t.head.idx < len(w) + wo + backtrack: t = t.head\n",
    "    return t\n",
    "\n",
    "# Returns subsequences of a name\n",
    "def subnames(name):\n",
    "    if type(name) != str: name = candidate_dict[name]\n",
    "    parts = name.split(' ')\n",
    "    subnames_ = []\n",
    "    for i in range(len(parts)): \n",
    "        for j in range(i + 1, len(parts) + 1): \n",
    "            sub = ' '.join(parts[i:j])\n",
    "            if len(sub) > 2: subnames_.append(sub)\n",
    "    return subnames_\n",
    "\n",
    "# Returns subsequences of a name unless potentially ambiguous (if another candidate picks out same subsequence)\n",
    "def nameset(name, candidate_dict):\n",
    "    if type(name) != str: name = candidate_dict[name]\n",
    "    subnames_ = [sn for sn in subnames(name)]\n",
    "    return [c for c in subnames_ if c not in sum([subnames(c) for c in candidate_dict.values() \n",
    "                                                  if c not in subnames_ and name not in subnames(c)], [])]\n",
    "\n",
    "# Given the original candidate dict and the final candidate list, returns new dict grouping putative candidate instances under a single key\n",
    "def candInstances(candidates, candidate_dict):\n",
    "    candidates_by_name = {}\n",
    "    for c in sorted(candidates, key = lambda c: len(candidate_dict[c]), reverse = True):\n",
    "        name = candidate_dict[c]\n",
    "        for name2 in candidates_by_name.keys():\n",
    "            if name in nameset(name2, candidate_dict): name = name2; break\n",
    "        candidates_by_name[name] = candidates_by_name.get(name, []) + [c]\n",
    "    return candidates_by_name\n",
    "\n",
    "import gender_guesser.detector as gender # oops\n",
    "gd = gender.Detector()\n",
    "\n",
    "# Needed to prune candidate dict-- removes non-provided candidates that don't match in most common gender with pn\n",
    "def filterGender(candidates_by_name, a, b, pn):\n",
    "    badnames = []\n",
    "    gender = 'female' if pn in ['She', 'she', 'her', 'Her'] else 'male'\n",
    "    for name in candidates_by_name.keys():\n",
    "        if a in subnames(name) or b in subnames(name): continue\n",
    "        genderii = gd.get_gender(name.split(' ')[0])\n",
    "        if gender == 'male' and genderii == 'female': badnames += [name]; continue\n",
    "        if gender == 'female' and genderii == 'male': badnames += [name]; continue\n",
    "    for name in badnames: candidates_by_name.pop(name)\n",
    "    return candidates_by_name\n",
    "\n",
    "from urllib.parse import unquote\n",
    "import re\n",
    "\n",
    "# Authors' metric 1: Does the Wikipedia url contain the candidate's name?\n",
    "def urlMatch(a, b, url, candidate_dict):\n",
    "    url = re.sub('[^\\x00-\\x7F]', '*', unquote(url.split('/')[-1])).replace('_', ' ').lower()\n",
    "    return {'a_url': (sorted([len(n.split(' ')) for n in nameset(a.lower(), candidate_dict) if n in nameset(url, candidate_dict)], reverse = True) + [0])[0],\n",
    "            'b_url': (sorted([len(n.split(' ')) for n in nameset(b.lower(), candidate_dict) if n in nameset(url, candidate_dict)], reverse = True) + [0])[0]}\n",
    "\n",
    "# Authors' metric 2: When pn is subject or object, does the candidate match?\n",
    "def parallel(t1, t2):\n",
    "    if t1.dep_.startswith('nsubj'): return t2.dep_.startswith('nsubj')\n",
    "    if t1.dep_.startswith('dobj'): return t2.dep_.startswith('dobj')\n",
    "    if t1.dep_.startswith('dative'): return t2.dep_.startswith('dative')\n",
    "    return False\n",
    "\n",
    "# Depth from a node to a parent node\n",
    "def depthTo(t1, t2):\n",
    "    depth = 0\n",
    "    while t1 != t2 and t1 != t1.head:\n",
    "        t1 = t1.head\n",
    "        depth += 1\n",
    "    return depth\n",
    "\n",
    "# Syntactic distance within a single tree\n",
    "def nodeDist(t1, t2):\n",
    "    if t1 == t2: return 0\n",
    "    if t2 in t1._.d: return depthTo(t2, t1)\n",
    "    if t1 in t2._.d: return depthTo(t1, t2)\n",
    "    t = t1\n",
    "    while t1 not in t._.d or t2 not in t._.d and t != t.head: t = t.head\n",
    "    return depthTo(t1, t) + depthTo(t2, t)\n",
    "\n",
    "# Authors' metric 3: Syntactic distance (within or across trees)\n",
    "def synDist(t, pn, doc, debug = False):\n",
    "    doc_sents = list(doc.sents)\n",
    "    sspan = doc_sents.index(pn.sent) - doc_sents.index(t.sent)\n",
    "    if sspan == 0: # same sentence\n",
    "        dist = nodeDist(t, pn)\n",
    "    else: # different sentence\n",
    "        dist = nodeDist(pn, doc_sents[doc_sents.index(pn.sent)].root) + nodeDist(t, doc_sents[doc_sents.index(t.sent)].root) # dist from two roots\n",
    "    if debug: \n",
    "        print('pn dist:', nodeDist(pn, doc_sents[doc_sents.index(pn.sent)].root), '; t dist:',\n",
    "              nodeDist(t, doc_sents[doc_sents.index(t.sent)].root), '; span:', sspan)\n",
    "    sspan = abs(sspan) * 1 if sspan >= 0 else abs(sspan) * 1.3 # less local if not preceding\n",
    "    return dist + sspan# * 0.7\n",
    "\n",
    "# Character distance\n",
    "def charDist(t1, t2):\n",
    "    if t2.idx > t1.idx:\n",
    "        return t2.idx - t1.idx + len(t1.text)\n",
    "    else:\n",
    "        return (t1.idx - t2.idx + len(t2.text)) * 1.3\n",
    "\n",
    "# Theta prominence: assign a 0.1 to 1 score based on dep role of candidate -- strong feature\n",
    "def thetaProminence(t, mult = 1, debug = False):\n",
    "    while t.dep_ == 'compound': t = t.head\n",
    "    if debug: print('t dep_:', t.dep_)\n",
    "    if t.dep_ == 'pobj': mult = 1.3 if t.head.i < t.head.head.i else 1\n",
    "    if t._.domain.dep_ == 'advcl': mult = 1.3 if t.head.i < t._.domain.head.i else 1\n",
    "    if t.dep_.startswith('nsubj'): score = 1\n",
    "    elif t.dep_.startswith('dobj'): score = 0.8\n",
    "    elif t.dep_.startswith('dative'): score = 0.6\n",
    "    elif t.dep_.startswith('pobj'): score = 0.4\n",
    "    elif t.dep_.startswith('poss'): score = 0.3\n",
    "    else: score = 0.1\n",
    "    if debug: print('mult:', mult, '; score:', score)\n",
    "    return min(1, score * mult)\n",
    "\n",
    "# Computes these metrics for each candidate, and returns, for each group of instances (A instances, B instances,\\\n",
    "# other instances), either the sum, or the highest difference from the mean\n",
    "def score(label, candidates_by_name, a_cand, b_cand, func, minsc = None, method = 'sum'):\n",
    "    if method == 'sum':\n",
    "        scores = {name: sum([func(t) for t in tokens]) for name, tokens in candidates_by_name.items()}\n",
    "    elif method == 'meandiff':\n",
    "        mean = np.mean(sum([[func(t) for t in tokens] for tokens in candidates_by_name.values()], []))\n",
    "        scores = {name: mean - min([func(t) for t in tokens]) for name, tokens in candidates_by_name.items()}\n",
    "    sca = scores[a_cand] if a_cand else minsc\n",
    "    scb = scores[b_cand] if b_cand else minsc\n",
    "    screst = [v for n, v in scores.items() if n != a_cand and n != b_cand]\n",
    "    if method == 'sum':\n",
    "        screst = sum(screst) if len(screst) > 0 else minsc\n",
    "    elif method == 'meandiff':\n",
    "        screst = max(screst) if len(screst) > 0 else minsc\n",
    "    return {'a_' + label: sca, 'b_' + label: scb, 'n_' + label: screst}\n",
    "\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "# Load a rowfull of data\n",
    "def load_row(data, i):\n",
    "    return tuple(data.iloc[i])\n",
    "\n",
    "# Row by row, populate features\n",
    "def annotateSet(data, minsc = None, debug = False):\n",
    "    \n",
    "    annotated_data = pd.DataFrame() # init placeholder df\n",
    "    row_batch = []\n",
    "\n",
    "    for i in tqdm(range(annotated_data.shape[0], data.shape[0])):\n",
    "        id, text, pn, pno, a, ao, ag, b, bo, bg, url = load_row(data, i)        \n",
    "        doc = nlp(text) # parse text into doc\n",
    "        pnt, at, bt = (doc._.to(pno), find_head(a, ao, doc), find_head(b, bo, doc)) # get the tokens that correspond to offsets\n",
    "        candidate_dict = {e.root: re.sub('\\'s$', '', e.text) for e in [e for e in doc.ents if e.root.ent_type_ == 'PERSON']} # first get every PERSON ent as candidate\n",
    "        candidate_dict.update({c.root: re.sub('\\'s$', '', c.text) for c in doc.noun_chunks if c.root.pos_ == 'PROPN' and c.text in sum([subnames(n) for n in candidate_dict.values()], []) and\n",
    "                               c.root not in candidate_dict.keys()}) # get some missed ones by looking at noun chunks with PROPN roots whose text match part of a candidate but are not already in list\n",
    "        candidate_dict.update({t: w for t, w in [(at, a), (bt, b)]}) # add provided cands, overwriting in the process\n",
    "\n",
    "        candidates = disq(pnt, list(candidate_dict.keys()), candidate_dict, debug = False)\n",
    "        candidates_by_name = candInstances(candidates, candidate_dict)\n",
    "        candidates_by_name = filterGender(candidates_by_name, a, b, pn)\n",
    "        a_cand = ([name for name, tokens in candidates_by_name.items() if at in tokens] + [False])[0]\n",
    "        b_cand = ([name for name, tokens in candidates_by_name.items() if bt in tokens] + [False])[0]\n",
    "    \n",
    "        # init row dict\n",
    "        features = {'id': id, 'label': 0 if ag else 1 if bg else 2}\n",
    "        # eliminated or not\n",
    "        features.update({'a_out': 0 if a_cand else 1, 'b_out': 0 if b_cand else 1})\n",
    "        # url match or not\n",
    "        features.update(urlMatch(a, b, url, candidate_dict))\n",
    "        # c-command or not\n",
    "        features.update({'a_cc': 1 if a_cand and pnt in at._.ccom else 0, 'b_cc': 1 if b_cand and pnt in bt._.ccom else 0})\n",
    "        # parallelism score\n",
    "        features.update(score('par', candidates_by_name, a_cand, b_cand, lambda t: parallel(t, pnt), minsc = minsc))\n",
    "        # theta prominence score\n",
    "        features.update(score('th', candidates_by_name, a_cand, b_cand, thetaProminence, minsc = minsc))\n",
    "        # syntactic distance score\n",
    "        features.update(score('loc', candidates_by_name, a_cand, b_cand, lambda t: synDist(t, pnt, doc), method='meandiff', minsc = minsc))\n",
    "        # number of candidates left\n",
    "        features.update({'n_cands': len(candidates_by_name)})\n",
    "        # char dist\n",
    "        features.update(score('cloc', candidates_by_name, a_cand, b_cand, lambda t: charDist(t, pnt), method='meandiff', minsc = minsc))\n",
    "\n",
    "        if debug: print(id, '>', 'a:', 1 if ag else 0, 'b:', 1 if bg else 0, features)\n",
    "        row_batch += [features]\n",
    "\n",
    "    # add rows to placeholder df\n",
    "    if annotated_data.shape[0] != data.shape[0]: annotated_data = annotated_data.append(row_batch, ignore_index = True)\n",
    "    \n",
    "    return annotated_data\n",
    "\n",
    "# Readable rows\n",
    "\n",
    "from textwrap import TextWrapper\n",
    "wrapper = TextWrapper(width=75)\n",
    "\n",
    "def style(w, wstyle):\n",
    "    if wstyle == None: return '{}{}{}'.format('\\033[1m', w, '\\033[0m') # bold\n",
    "    elif wstyle == True: return '{}{}{}'.format('\\033[92m', w, '\\033[0m') # green\n",
    "    elif wstyle == False: return '{}{}{}'.format('\\033[91m', w, '\\033[0m') # red\n",
    "\n",
    "def readable_rows(data, range):\n",
    "    rows = []\n",
    "    for i in range:\n",
    "        wois = sorted([(data.iloc[i]['Pronoun'], data.iloc[i]['Pronoun-offset'], None),\\\n",
    "                   (data.iloc[i]['A'], data.iloc[i]['A-offset'], data.iloc[i]['A-coref']),\\\n",
    "                   (data.iloc[i]['B'], data.iloc[i]['B-offset'], data.iloc[i]['B-coref'])],\\\n",
    "                   key = lambda x: x[1]) # sort by offset\n",
    "        text = ''; ftext = data.iloc[i]['Text']; coffset = 0\n",
    "        for w, woffset, wstyle in wois:\n",
    "            text += ftext[coffset:woffset] + style(w, wstyle)\n",
    "            coffset += len(ftext[coffset:woffset]) + len(w)\n",
    "        text += ftext[coffset:]\n",
    "        rows += [(str(i), data.iloc[i]['ID'], 'A' if data.iloc[i]['A-coref'] else 'B' if data.iloc[i]['B-coref'] else 'N', text)]\n",
    "    return rows\n",
    "\n",
    "def print_readable_rows(data, range):\n",
    "    for index, id, target, text in readable_rows(data, range):\n",
    "        text = '\\n\\t'.join(wrapper.wrap(text = text))\n",
    "        print('{} ({}): {}>\\n\\t{}\\n'.format(id, index, target, text))\n",
    "        \n",
    "def print_tokens(doc):\n",
    "    for i in range(0, len(doc), 3):\n",
    "        for t in doc[i:i+3]:\n",
    "            print(\"{}[{}] >{}> {}\".format(t.text, t.pos_, t.dep_, t.head.text), end = ' | ')\n",
    "        print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Widget Javascript not detected.  It may not be installed or enabled properly.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "837dd9e436964449bb7f955d3823b713"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\numpy\\core\\fromnumeric.py:2957: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\numpy\\core\\_methods.py:80: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "annotated_data = annotateSet(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "As wrongly disqualified: 57 , Bs wrongly disqualified: 94 \n",
      "\n",
      "development-155 (154): A>\n",
      "\tThe official first single, ``Fallin''', was written solely by Keys, and\n",
      "\ttopped the US Billboard Hot 100 chart. ``A Woman's Worth'', written by\n",
      "\tKeys and Erika Rose, is a ``jazz-tinged'' song with\n",
      "\tlyrics which speak of how men should treat and respect women. Keys released\n",
      "\ther second album, The Diary of Alicia Keys, in December 2003.\n",
      "\n",
      "The[DET] >det> single | official[ADJ] >amod> single | first[ADJ] >advmod> single | \n",
      "single[ADJ] >nsubjpass> written | ,[PUNCT] >punct> single | ``[PUNCT] >punct> single | \n",
      "Fallin[PROPN] >appos> single | '[PUNCT] >punct> Fallin | ''[PUNCT] >punct> single | \n",
      ",[PUNCT] >punct> written | was[VERB] >auxpass> written | written[VERB] >ROOT> written | \n",
      "solely[ADV] >advmod> written | by[ADP] >agent> written | Keys[NOUN] >pobj> by | \n",
      ",[PUNCT] >punct> written | and[CCONJ] >cc> written | topped[VERB] >conj> written | \n",
      "the[DET] >det> chart | US[PROPN] >compound> chart | Billboard[PROPN] >nmod> chart | \n",
      "Hot[PROPN] >nmod> chart | 100[NUM] >nummod> Hot | chart[NOUN] >dobj> topped | \n",
      ".[PUNCT] >punct> written | ``[PUNCT] >punct> Worth | A[DET] >det> Woman | \n",
      "Woman[NOUN] >poss> Worth | 's[PART] >case> Woman | Worth[PROPN] >nsubj> is | \n",
      "''[PUNCT] >punct> Worth | ,[PUNCT] >punct> Worth | written[VERB] >acl> Worth | \n",
      "by[ADP] >agent> written | Keys[PROPN] >pobj> by | and[CCONJ] >cc> Keys | \n",
      "Erika[PROPN] >compound> Rose | Rose[PROPN] >conj> Keys | ,[PUNCT] >punct> is | \n",
      "is[VERB] >ROOT> is | a[DET] >det> song | ``[PUNCT] >punct> song | \n",
      "jazz[NOUN] >npadvmod> tinged | -[PUNCT] >punct> tinged | tinged[VERB] >amod> song | \n",
      "''[PUNCT] >punct> song | song[NOUN] >attr> is | with[ADP] >prep> song | \n",
      "lyrics[NOUN] >pobj> with | which[ADJ] >nsubj> speak | speak[VERB] >relcl> song | \n",
      "of[ADP] >prep> speak | how[ADV] >advmod> treat | men[NOUN] >nsubj> treat | \n",
      "should[VERB] >aux> treat | treat[VERB] >pcomp> of | and[CCONJ] >cc> treat | \n",
      "respect[VERB] >conj> treat | women[NOUN] >dobj> respect | .[PUNCT] >punct> is | \n",
      "Keys[NOUN] >nsubj> released | released[VERB] >ROOT> released | her[ADJ] >poss> album | \n",
      "second[ADJ] >amod> album | album[NOUN] >dobj> released | ,[PUNCT] >punct> album | \n",
      "The[DET] >det> Diary | Diary[PROPN] >appos> album | of[ADP] >prep> Diary | \n",
      "Alicia[PROPN] >compound> Keys | Keys[PROPN] >pobj> of | ,[PUNCT] >punct> album | \n",
      "in[ADP] >prep> released | December[PROPN] >pobj> in | 2003[NUM] >nummod> December | \n",
      ".[PUNCT] >punct> released | \n",
      "\n"
     ]
    }
   ],
   "source": [
    "misdisqs_a = annotated_data.loc[(annotated_data['a_out'] == 1) & (annotated_data['label'] == 0)]\n",
    "misdisqs_b = annotated_data.loc[(annotated_data['b_out'] == 1) & (annotated_data['label'] == 1)]\n",
    "\n",
    "print('As wrongly disqualified:', misdisqs_a.shape[0], ', Bs wrongly disqualified:', misdisqs_b.shape[0], '\\n')\n",
    "\n",
    "for i in misdisqs_a.index[:1]: # the first\n",
    "    print_readable_rows(data, [i])\n",
    "    id, text, pn, pno, a, ao, ag, b, bo, bg, url = load_row(data,i)\n",
    "    doc = nlp(text); print_tokens(doc)\n",
    "    print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x2db1140d048>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEVCAYAAAAM3jVmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xm4HFWd//F33yXcBLIASUhwIYj0\nF9DRMSMQAQdUBEEExtGRYRwFQZZBVvnNjwEUVASHGUEZEcRR3EBcHh1gJIAMiwlhURF+rF+Gh0VE\nEkgICWS/t/v3R1WTpul7u6pO9Xb783qeenK7q759TvrWPX361DnfKpTLZUREpDf0tbsCIiLSOmr0\nRUR6iBp9EZEeokZfRKSHqNEXEekhavRFRHrIQLsrkLdjCnOC5qBe/MR/ZY4dfvLBkKJh7n5B4eWF\nPwmKL73n8Myxmyx5JKjsDTO3D4rvf+m5oPjVm80Kih8aWZM59uXCUFDZm/aHTbteNVIIil++diRz\n7PSJ/UFlT+gP67dOmjgU9p8nXZtzafnJ4PJCjbtGX0Sklfrb3oyno0ZfRCRAf6G7Wn01+iIiAdTT\nFxHpIerpi4j0EPX06zCzKcB/AtOA6cC33f2SUY49Ezg4rtsl7v6tes+1ot4iIo10W0+/VfP03wxc\n5e77AAcAp9Q7yMzeAewH7ArsBuw0ynPd9S6LyLg1oa+QeOsErRreWQycZGYfBlYCg6McZ8Dd7j4C\nrAZONLNDap9rRYVFRJLothWurWr0TwXucPdLzOw9wAdHOe4R4Fgz6wP6geuAM+s8d4C7r2tBvUVE\nxqThnfquJeq1LwROAobNbJPag9z9XuB64HZgIXCFu99V5zk1+CLSEfoLybdO0JKevrvfAuyQ8Njz\ngPMaPSci0gm6rafflimbZnYUcGidXf/i7ne0uj4iIll1ygXapNrS6Lv7ZcBlzXjtkIRpAMdte3Dm\n2K/f8uWgsgul4aD48sCEoPhVG0qZYwcnTg0qu/Sri4Pi+16/XVD8pO3+Kii+8HT2ZHuT37xrUNml\nBT8Lit9sz48HxU8eWJU5tvBy9liAkXtuDIpn/2PD4umcYZuktDhLRCSAhndERHqIevoiIj2k2xr9\ntq8rMLM5ZnZnu+shIpJFf6GQeOsE6umLiATIa/ZOvAD1m8DbgXXAke7+WNX+U4G/B0rAue7+yyzl\n5NLohyRUA26o2vd+4BxgLbAM+BRR2oaLgF2ACcBZ7n51HvUWEQmV4/DOwcCQu7/LzOYBXwUOAjCz\nacAJRHnMNgXuBTI1+nkN72ROqAYU4n0FommcH3b3PYHbiFIwHARMd/ddgA8AO+dUZxGRYDkO7+xB\nlH0Ad78TeGfVvlXAU0QN/qZEvf1M8hreCUmoNifeNx1Y6e7PxI9/A5wLLAXuAHD3xUQfBCIiHSHH\nnv4UYEXV4xEzG3D3ygKep4GHiHKQZc5QkFdPv5JQ7ePAz4h773U8Asw1sz4zGzSzXwOVHDxLgSlm\nNjt+vCfwKPAwce/ezKaa2Q21Lyoi0i459vRXApOrHvdVNfj7AbOBbYE3Ageb2S5Z6ptXo585oRrR\nBQvcvQx8GviFmd0O7A18CbgGWB6/9g3A13Kqs4hIsL5CIfHWwO3A/gDxmP79VfuWA2uAde6+FniR\n6BpqarkM74QmVAPmxftuAm6qE3Z8UAVFRJqkf0JuM99/CbzfzBYRjZYcbmanAI+5+zVmtjdwp5mV\niDrNv85SSFOmbCqhmoj0ikJ/Po2+u5eAY2qefqRq/1nAWaHlFMrlcuhrdJR1t17Rtv/Qie85Iyj+\nG/dcGhS/5v6wNW6Tdnlf5ti7hnYKKnvniSuD4p8sTA+Kn7xJ2B/uuuHsp936kbBTdruV2ZO9AQzP\neHNQ/POliZljt3zNIHA6/SsXB8UPztou+DLsDcW5iX+B+z56T9tXaGlxlohIgL4uy8OgRl9EJECh\nr+3ZbFJRoy8iEiDHC7kt0fbaKuGaiHSzQn9f4q0TqKcvIhJAY/qxNEnYqmKUcE1Eukqhy+6R28zv\nG4mSsFUo4ZqIdKO+/r7EWydo5vBO0iRsFUq4JiJdp9BlwzvN/OhJmoStQgnXRKTr9E/oT7x1gmb2\n9K8FLjGzfyAanx82s03cfV29g929bGaVhGslogRDh8Wxe8cJ1waALzSxziIiqXRbT79pjX7SJGzu\n/iRKuCYiXaqvyy7ktmzKppKwich41Cnz75NqWaPv7pcRzc5prrn7BYUXSsONDxrFN+7ZMqjsz8yt\nTbCXzufO3jcofuJfrMocO2OLCUFlE5gmbyDw7276qj8FxS/f7A2ZY0fKI0FlL5v5tqD4CYHDE8Nr\ns9d/cNkTQWW/MHW7oPgZQdERzdMXEekhGtMXEekhnTIrJyk1+iIiAbptRW7ujX7S9AtmNodo/v6z\nwOuB+e5+hpm9FbiAaA3BNOAEd19kZk8R3UXmYXc/Ke96i4hk0SkrbZNqRm3TpF+YQzQXf2fgvWY2\nF3gL8Fl335uo8T88PvYNwKFq8EWkkxT6C4m3TtCM4Z006Rfuc/cXAMzsLsCAp4HPmdkaYHL8GgBL\n3X1ZE+orIpJZt03ZbEZt06Rf2NHMJplZP7Ar8BBRRs2z3P2TwP1V8aUm1FVEJEj/4EDirRM0o9G/\nFjgxTptwEnH6hVGOXU/0wXAXcLW73wf8CLjazBYARWDrJtRRRCQXPX8TlaTpF2JL3P2DNfEXEI3l\n177urByqJyKSq05pzJNq+veNsdIvNLtsEZFm043RazRIvzCv2eWLiDRToV+Ls0REeoaGd9qsvPAn\nYfED2ROHrVv8dFDZoQnTvnR22P1l/mP/DzY+aBQzJoX1dgor1gbFb75pWPlrJ70xKH5ywBTsZWvC\nss0NDYTN/x4IXFE6Us5e//Uztg8qu39D+yf1dcqsnKS6q7YiIh1GPX0RkR6iC7k5iGf8XA7sDhzj\n7oe0uUoiInV1W0+/U2t7OtBdl8RFpCf1/OKs0aTIvnkEMAu4CvgasL2ZzQdmAte6+9mtqrOISCPK\nsjm6RNk33f07REnbKkM6Q8DBwLuBz7SgniIiifUNDiTeOkEra5Em+2a1B9x9HYCZZb+BrYhIE3TK\nsE1SraxtmuybJTbWLfCW2SIizVPo60u8dYJW1iJN9s0FwHWM/cEgItJ2ff39ibdO0LLhnTTZN+Nc\n+hW3VD2vTJsi0lHyGt4xsz7gm8DbgXXAke7+WJ1jfkWUiv7SLOW07crCWNk33f2OVtdHRCSLHC/Q\nHgwMufu7zGwe8FXgoJpjzgG2CCmkbY1+g+ybIiJdIccLuXsA1wO4+51m9s7qnWb2EaLrnfNDCumM\nOUQ5Kr3n8MYHjWFVQAKnLf78+6CyJ/7FqqD4kIRpAMfvckLm2JMW/7+gsmdPmxMU/8cV64Pip24S\nNt66dHX2iWWrN4wElT1zUtifceiFs80Gszd6qwMTpr28Piw+qMscy7HRnwKsqHo8YmYD7j5sZm8l\nGhn5CPD5kELGXaMvItJKOc7KWQlMrnrc5+6V3sQngNcBNwNzgPVm9qS7X5+2EDX6IiIBCn25zcq5\nHfgQ8NN4TP/+yg53/+fKz2Z2NrA4S4MPnZt75zXMbI6Z3dnueoiIvEpff/JtbL8E1prZIuBC4GQz\nO8XMDsyzuurpi4gEKAwmTS4wNncvAcfUPP1InePODimnqY1+iiRrc4jy8syLH99JlHvnMGA3YDPg\niGbWVUQkk/yGd1qi2cM7iZKsNfCwu+8GrMm1ZiIiechveKclmj28kzXJWvUsMs+9ViIiOemUnDpJ\nNbu2SZOsrQVmmlm/mU0Dtq3a1/47H4uIjKbLevrNbvQTJVlz98XAr4HfEq3Sfaz2GBGRjtRljX5T\nh3dSJlk7us7TZ1ftfxKYl0vFRERyUhjIZ/ZOq7R0yqaSrInIuNMhPfikCuXy+LpHyciT9wb9h0oT\np2aO/e2aKSFFM2PShMD4sJNv8ars+WO+NuttQWV/068Iin/i4v8Iin/jmecHxa++JnvuwM3e95Gg\nsoenb9v4oDGsGAnr+60MyH8zfWLYOTvUH5Y5aGjixOB7dqxfcFXiNmfCuw9p+z1CtDhLRCREl83e\nUaMvIhKiy4Z31OiLiATQhdw6zOwwYAd3P60V5YmItEqOWTZbQj19EZEQavRH9S4z+x+iu8Oc7e6/\nqj0gvunvRcAuwATgLKIFXq96zt2vblmtRUTG0mUXcltZ21XA3sAHgW/EDXytg4Dp7r4L8AFg51Ge\nExHpCIX+/sRbJ2hlT3+hu5eB58xsBbAl8HzNMQbcAa+kZjjTzE6rfa51VRYRaaDLhnda2dPfGcDM\nZhHlx19a55iHq46bamY3jPKciEhHKAwMJt46QSt7+hPN7GaiBv/ouNdf6xpg7zhB2wDwBeD6Os+J\niHSGLuvpt6TRd/fvAd9LcFwZOL7OrnrPiYi0X6G7LuS2ZcqmmX0eeG+dXYe7+xOtro+ISGZd1uiP\nu4Rra1evCku49quLM8cO/vVHQ4oOVhheGxS/atqczLGTn7knqOx/sn8Iiv/qlUcGxQ/tcVBQ/Po/\n3JI5tv/dfxdUdvn384PiV+36saD4iYPZG72BFc8Gld237Kmg+P6d9gpOgDbyxD2J25z+becq4ZqI\nSFfTmL6ISA/pssVZavRFRAKUu2xMP5famtlhZvaVjLF7mdlVedRDRKTlCn3Jtw6gnr6ISIgOacyT\nyrPRz5pQbUXV/n8ATgLWAf8LHBXX8XJgG2AQOF730xWRjtFljX6etc2aUA0AM9uSaLXte919D+BF\n4GjgGOBJd38XcBiwa451FhEJUu4bSLx1gjxrkTWh2l7xvjcBD7r7S/Hj3wD7AAVgfhzzAPBAjnUW\nEQlTaPvU+1Ty7OlnTahW8QSwk5ltGj/eE3i0JuZNZnZljnUWEQnTwxdysyZUA8Ddl5rZWcAtZlYC\nHgMqt1f8rpndBvQTjfmLiHSEbpuymUujn0NCtVvj/VcC9Xryh2avnYhIE2lxlhKqiUgP6bKe/rhL\nuLZhyRNB/6Hy49kThz213T4hRTMQeO5sPhSWA+SPKzdkjt30X48OKnvWvLcGxX/20P8Mir/4ybDb\nLq+87seZY6fu+5GgsikNB4Wv2vrtQfGLX85e/uzNwvqdQyv+FBQ/MHv74Kuw61csTdzmTJg6ve1X\nfTtjDpGISJfqyTF9EZGelVOjH69t+ibwdqIFqke6+2NV+z9NtHZpGDjH3f87SzlN+4hKmo9HuXdE\npKsVCsm3sR0MDMULUU8DvlrZEU+FPwHYHdgXOM/MNslS3e76XiIi0mnym6e/B9E9wXH3O4F3Vu3b\nBbjd3de5+wqiKe1vy1LdZg/vNMzHU025d0Sk2+SYXmEKVbnIgBEzG3D34Tr7XgKmZimk2T39JPl4\nAOXeEZHuVC4UEm8NrAQmVz3uixv8evsmE7WRqTW7p58kH0+Fcu+ISNfJcdb77cCHgJ+a2Tzg/qp9\ndwNfNrMhYBNgRzK2hc3u6SfJx1Oh3Dsi0nVK5XLirYFfAmvNbBFwIXCymZ1iZgfGCSovAhYANwNn\nuPvaLPVtdk8/ST4eQLl3RKQ75dXRd/cS0XB2tUeq9n8b+HZoOU1r9FPk47kV5d4RkS5V6rKkBi1b\nnKV8PCIyHo10Wavfskbf3b8IfLFV5YmItEJ3NfnjMOHailVrgv5Dk1YtyRy7bJMZIUUzfVVY8qi1\n094YFL9szUjm2K1LLwSV3bd2ReODxlIqBYUfN+egoPgv/lv2+C3/9pNBZa+YvkNQ/IT+sBxga4az\n/8lt/vLTYWUHnvOTJ00MToD2/MrVid+AGVMmKeGaiEg367aOc+5TNpPm3BERGQ9KKbZOoJ6+iEiA\nLuvoN63Rb5hzx8z2As4g+gCcBVzm7heb2Z7AWfFhk4BPAOuBa4FlwHXufn6T6i0ikkq3zd5p1orc\npDl3XgccCMwjWn02E3gL8HF3fy/RjdQ/Gh87C9hHDb6IdBIN70SS5txZ5O7rAMzsAWA74BngIjN7\nmehD4fb42CfcfX2T6isikomGdyJJc+78pZn1EyUQegtROuVrgDe5+0tm9n2ihGvQOR+UIiKvSJBT\np6M0q9FPmnNnkCiD5pZEt/9aamY/BO4ys+XAEmDrJtVRRCRYdzX5TWj0k+bciT3s7ofUxJ8CnFLn\n2HlhNRMRyd9Il41BNH3K5hg5d77f7LJFRJqt1GV9/aY3+g1y7lze7PJFRJqpy4b0tThLRCREl03T\nH3+N/tDImqD4wtMPZo5dN2fPoLKXb/aGoPjJgamclq4ebnzQKKbdeFlQ2RNmzAyKX/vMn4PiQxKm\nAXz+/1ydOfbrc/8qqOyJs3cKig9dXDQcEL9+822Cyi50QDe7A6qQyrhr9EVEWklj+iIiPURpGMaQ\nJgOnmX0mbYyISKuNlJJvnaCljX5KZ7a7AiIijZTK5cRbJ2jH8E6SDJxnAFuY2TeBu4F5ZnYjMAO4\nxN3DrhqKiORkpEMa86Ta0dNvmIHT3b8MvODu/xQ/tQHYF/gb4KRWVVREpJFu6+m3o9Ff6O5ld38O\nqGTgbOSeOH/PYqIc+yIiHaHbxvTbMbyTNANn9azzzviIFBGpsaHUIa15Qu1o9JNm4HzIzH4E3NS6\nqomIpNNlMzZb2+inycDp7u+p89xaYE6ulRIRCdBt8/TbujhrjAych7v7E62uj4hIWp1ygTapQrnL\nKtzIspdWB/2HJpdWZ459at0mIUXTH3hZff1I2O/y+VUbMsfu1vd0UNkjU2cFxQ8s/1NQfLkQ9uYP\nP/FA5tgT3/f5oLLPXflQUHx/X1jSppDcO4Hpolgf2MueNXXT0Crw3w8vSVyJA3bcKri8UErDICIS\nYEOnTMtJSI2+iEiAbhveUaMvIhIgcFS15Zq+OCvPhGlmtpeZXZXHa4mI5KHbVuSqpy8iEqCkKZt1\nJUmythdwjLsfEj9e7O6zzOx7RKkatgT+rUX1FRFJRMM79TVMstbAze6+G7A895qJiATYUCol3jpB\nq3r6C+N0C8+ZWSXJ2vMNYqrns3rTaiYiEqCZwztmNhH4ETATeAn4pLu/pu00s0nAIuA0d79+rNds\nVU8/SZK1tcDs+LhtgC2q9nXGR6SISI2RcvItg2OB+9393cAPGP3mUheTMDFlq3r6SZKs/Q540czu\nAh4GlIZBRDpek2fl7AGcH/88H/hc7QFmdipRLz/Rat+mN/pJk6y5+zBwUJ3nD6v6+Vbg1rzqJiIS\nKq87Z5nZEcDJNU8vIbrvCETDO1NrYt4HbO/uR5vZ7knKafmUTSVZE5HxZP1wPqPP7v4d4DvVz5nZ\nL4DJ8cPJwIs1YUcA25jZrcAOwNx45uO9o5XT8kbf3b8IfLFZr79pf9inbmnBzzLHbld8R1DZy2a+\nLSh+aCAsl9PMSdlPh+HCtkFlc+cvwuLn7BgUvmKrsPd+4uydMseeu/IjQWWfPiV72QBf+dahQfET\nhiZkjp34vo8FlX3PEccFxc+6/uageGh6auXbgf2J7hW+H7Cgeqe7v/LLi6e3XzVWgw9anCUiEqTJ\njf4lwPfNbCGwHjgUwMzOB37u7nenfUE1+iIiAZrZ6Lv7auCjdZ7/5zrPHZbkNTu20Y/z9TwSXwgW\nEelIunOWiEgPGXeNvpkdRnQhYRKwHfCvo/W+zexM4OD4dS9x92+Z2XnAO4muPD/s7oeb2dnAtkSr\nzLYBTnb3G8zsb4kWHzwPTAAeMbMZwE+IFpINEuXnuT/z/1hEJEd5zd5plaQrcqe6+wHAgcBp9Q4w\ns3cQXV3eFdgN2MnMpgLL3f398XPzzOx1ccg6d98POJGNc1PPJ8rRsy9QuW/hLkTzVPcDTiBK2iYi\n0hFGSuXEWydIOrxTmQL0NDA0yjEG3O3uI0QN9olmNgjMNLMfAy8TrcgdjI//Q/VrmtlWwEp3XwZg\nZovi/fOB7YGrgQ3AOQnrLCLSdCH3CG6HpD39JP+rR4gWBvSZ2aCZ/ZpoWOgN7v73wOnARDYuFa59\nzWXA1Hg4B+J8PcBewLPuvg9Rg39uwjqLiDRdt/X0c0u4Fi8IuJ5oMcFC4ArgLuBNZnYn8HPgcWDr\nUeKHgcOBG8zsJqIxfYD7gE+b2R1E+fTPy6vOIiKhuq3Rbzi8U33R1t3XAnPGOPY8Xtso71zn0Nur\nYh4h6s3j7rcBc+scv3ejeoqItENeuXdaJfWUTTM7inhVWI1/cfc7wqskItI9um32TupG390vAy5r\nQl1ERLpOpwzbJDXuFmetGglLOrbZnh/PHDs8vC6o7An9YXUf6AuLD4lesSHsVBrYNSzx1mDge5c9\nZVgk5A+/P/D3Fpow7bSjrwyKP/W4XTLHbvOBiUFlFz88Lyg+DyMdchvEpMZdoy8i0krq6YuI9BA1\n+imZ2RyiHNDt/54mIpLSuvF+IVdERDbqyZ5+SFI24Iaqfe8nWnW7lmiF7qeAlcBFRDl4JgBnufvV\nedRbRCRUtzX6ua3IJWNSNuJJI2ZWIJoK+mF33xO4jSjj5kHAdHffBfgA9Rd7iYi0xbhbkZtC1qRs\nc+J904kSrj0TP/4NUZ6dpcAdAO6+mOiDQESkI3RKY55Unj39rEnZNon3LQWmmNns+PGewKPAw8S9\nezObamY31L6oiEi7lEvlxFsnaOmFXHe/18wqSdn6iMb018X7ymb2aeAXZlYClgOHEY3t7x3fGHgA\n+EIr6ywiMpaRkR6cvZNDUrZ58b6bgJvqhB0fXEkRkSbolB58Uk3p6Sspm4j0ipIafSVlE5HeUe6u\n0Z3xtzhr+dqRoPjJA6syxz7P5KCyhwPrHprXe7PB7Nf1V64PO/NfN3mw8UFj+NPKDUHx04b6g+JD\nbpkXmmhvwlBYuriQhGkA/37x3ZljLz71xaCyN6xaExSfh/J4z6cvIiIbaXhHRKSHlLos906e8/Qz\nMbM58T10RUS6TqlcTrx1AvX0RUQCaMpmLE0StqoYJVwTka7SbY1+s4d3GiZhq1DCNRHpRqVSOfHW\nCZo9vJMkCVuFEq6JSNcpjXRGY55Us3v6ad4NJVwTka6jnn5GSrgmIt2o28b0m9boJ03C5u5PooRr\nItKl1OiPQknYRGQ86pT590kVui1vRCMvvrw66D+06cvPZo4dmTIrpGgGlz0eFL9+xvZB8as3ZF9Z\nGJg+hk1XLQmKXzc57L0fevGPQfHrN98mc+zawBWdU1Y+FRRfHpwYFN+3Onv+nOO2PTio7K98q14/\nMrlpR50beObCDidenbjNeeTrB6Uqz8wmAj8CZgIvAZ909+drjrkA2AMoAZ9199vHes22r8gVEelm\nI8OlxFsGxwL3u/u7gR9QM3vRzN5OdL/xXYF/JFrTNCY1+iIiAcrlcuItgz2A6+Of5wN71+x/huh+\n45sAU4CG6WY7ZvaOiEg3yutCrpkdAZxc8/QSYEX880vA1Jr9w0TDOo/E+z7dqBw1+iIiAfKaf+/u\n3wG+U/2cmf0CXrlRx2Sg9gLKJ4DFwL7x/oVmdkfVItfXyL3RT5pzx8zmAD8DngVeD8x39zPM7K3A\nBURDT9OAE9x9kZk9RfRp9rC7n5R3vUVEsiiXwm5+1MDtRO3p3cB+wIKa/cuBl919xMxeAtYBm431\ngs3q6U91933NbHvgWuB7oxw3h+gTagXRJ9RcYHuiK9D3m9mhwOHAIuANwFx3X9akOouIpNbkRv8S\n4Pvx4tT1xNPezex84OfAlcDuZrYI6AeucHcf6wWb1egnzblzn7u/AGBmdwEWx3zOzNYQfV1ZGR+7\nVA2+iHSa0ob1TXttd18NfLTO8/9c9fCYNK/ZrNk7SQe5djSzSWbWTzTl6CGiKUdnufsngfuByrzW\n7ro9jYj0hHJpJPHWCdp9IXc90bj+VsDP3f0+M/sRcLWZLQH+RJR9U0SkI3VKY55U7o1+0pw7sSXu\n/sGa+AuILuTWvm7YkksRkSbo+Ua/1lg5d5pdtohIs6nRr+HulxHdEaueec0uX0SkmUaGm3chtxna\nPaafuwn9YdemR+65MXNs/9x9gsp+Yep2QfH9AQnTAF5enz1+q0n9QWX3LQtLGjZUGg6KXzPtjUHx\nhYDEhesDF/fcc8RxQfHFD4f1vTasWpM5NjRh2mlHXxkUf+lR5wbFg3r6IiI9RY2+iEgPKY+o0RcR\n6Rnq6ecgnvFzObA7cIy7H9LmKomI1KVGPx+nE90wQESko5WGG6aw7yitvEfuYSTLvnkEMAu4Cvga\nsL2ZzSe6Xdi17n52i6osItJQt/X0W33nrKnufgBwIHBavQPinNKLgcqQzhBwMPBu4DOtqKSISFLK\nvTO2pNk3qz3g7usAzCxsMraISM5KHdKYJ9XqRj/pKpQSG7+F5HNbGhGRJui2KZudemP0BcB1bEyr\nLCLSkUrD6xNvnaBlPf002TfjXPoVt1Q9r0ybItJROmWsPqm2TdkcK/umu9/R6vqIiGTRbY1+oRyQ\nKEpERLpLp47pi4hIE6jRFxHpIWr0RUR6iBp9EZEeokZfRKSHqNEXEekhavRFRHpITzb6ZjbY7jok\nZWYXtLsOvcrMTm13HXqVzvvm6dSbqOTOzD4N7OTuJwO/MrMfuvsPU8R/CNjZ3T9vZtcDF7j7jS2I\n39HMprn7i0nLqlP2WURpqTcQ5TMqu/vWCeLu4LUJ7yrxu6UoP/S92xXY1d0vMrMrgK+6+z3NjgX2\nN7ML3T3zksv4vDsJmMjG9+5NCeJ+zCjJBt293kr2eq/xFmAKUQLDc4Fz3f1/Ela9nec8BJ73Wc/5\nXtAzjT5wLFBpqD4I/AZI3OgDXwA+EP/8MWA+kPgPICB+J2CZmT1P1AhkOXkPAN7o7mtSxlXuaTAR\nSBtbLfS9+w/gsPjnzwHfA/66BbEzgD+b2RNsfO8Tf9jFjiG6edDilHGXxv9uDixPGVv9GicSvf9n\nAOcDiRt92nfOQ/h5n/WcH/d6qdEfiRO94e4bzCxt/okN7v5cHL/CzNL2/jLFu/s2Kcup5zmiHk8q\n7v4UgJktdPc9AsoPfe+G3f2hOP5xMyu1KPaANJUcxdLK+5iGu98Gwe/9BuBBYIK732lmaf/e23LO\nx8eHnveZzvle0EuN/tVmtgC4G5gLXJMy/m4zuxK4A9gF+EMr4s1sHnA4MEj0NXVrd983YWxliGAr\n4A9m9kD8OPEQQWyVmV0IONFhcIr3AAAJoElEQVRQAe5+WYr40PfuKTM7tyr+mRbFDgIfpeq9B45O\nEhiXCTDBzG4A7mHje396ijq8YGYn8ur3PmlvuQxcCVxnZn8HrEpRLrTpnIfs532O5/y41TONvruf\nY2b/DRhwlbv/NuVLnAAcBBSBn7r7tS2Kvwi4EPgIcD8wIUWZlzY+JJFF8b9bZYyv/N+NbO/d4UTD\nJPsBDwPntCj2B8C1wB7An4HNUsR6zb8Vab9hLgP+Mt4q8Ukb/Y8RNbbzgb3ix2m065yH7Od9Xuf8\nuNUzjX71hVwzuzHthVxgMrA78BZgtpnd7u4vtCD+RXf/sZnt4+5nm9ltKcpcCPQT3WT+Y0Q9pn7g\nV8B7U7zO5SmOrecAYG7lgp6ZrUtzQZDoa3rlq3of6RrOkNjV7n6emW3v7p+Kvykm4u7fBzCzb7j7\nK/d2NrMfEH2YJH2dw1PUt1LGJ2qe+sf43zekKZv2nfOQ/bzP65wft3qm0Sf8Qu53gduAK4A9iS4I\nHtiC+HI8C2OSmRmQ5kYynwJOj2MqPc4S0R9GGj8haiz7gG2B/yXq/SYVekHwMuDFOGZP4D+B2oat\nGbEFM5sFTDazTYEtklbYzI4DzgQ2N7MPV14PeCjpa8Sv8yzRe1+Iy3/c3XdsEFbZPw9YTfRNbWei\noZI0jX67znnIft7ndc6PX+VyuSe2YrH425rHi1LG31LzeEEr4ovF4luKxeLfFYvF3YvF4u+LxeLJ\nGf7vnxrl+V0zvNa0YrH4k5QxdwW+d7/J+rsLjP3rYrF4bLFYPLBYLD5XLBb/PcP7dfooz2+T4bW2\nKRaLl6c4/vqaxzemLO+WwN9b5vjQ8z7Pc368bb20OOtqM1tgZl81s1uAq1PGT4x7fZjZVkRfGZse\n7+4PAo8CrwcOc/cLU5aLu393lF3npX0tYAWwXcqYu83sSjM7Ph7eSHtBcMjMJgGY2UTSvfeZY939\nN8DPgCXADu6eerGWu587yq7UQ2bxLKAdUoTMNLNpAGa2JbBlyiLbcs5D+Hmf8zk/rvTM8E7Nhdwf\nuPt9EC3ecfe7ErzE54BFZraSaKzyqJRVyBRvZl8iGou8GzjRzH7p7v+WsuzRJLrxfM0irZnAr1OW\nU30h9+fufk38utsknM74deC+eCbGTsDZKcrOHGtm/wScDDwA7GRmX3L3H6UoeyxJ3/vqRVqziT6A\nkvoy8Lv4nJtCNPSRRlvOeWjqeZ/ofR/X2v1Vo91bsVi8OeXx0wPLSxVfLBZ/VywW++Kf+4vF4t2t\n/r/HwwqVbauq54O+Kqd574vF4hbFYvGdxWJxywzlZIotFot/KBaLQ/HPk2qHCFv03u9Zte1aLBb7\nK7+TFGXNrMRlrGtLz/k4pinnfdq/9/G49UxPfwxjfvLXS0UQXVeCJKszQ+OBPxH1klYQXYhL09PL\nxRi98fMImxHR6L2vm4rAzBrOuQ6JrbIEGI5/XkM0fbKlKou06ricBu+9mb2f6JvKUPwYd2/4++qA\ncx464Lwfr9ToN57Cd0jVz4X4+E2AdQlf/5CxdiYYXtoaeNTM7iManlhvZosg1R9QbZmD7l7JSRIi\nNL7Rex8y53rM2IRDS33AvfH7/Q5gMF5slHqhj5lt6e7VHxqteO8vJMr783TK1273OQ9NOO9jPT+8\no0a/gapUBK+a50803bPhlM8EDUuj3vJHR3l+dqOyK8zsGOAUot93gWjeepFotWaItAuNUqlKRVCb\nuKvhRb0xesgVDXvKRGPiFVdU/TynUfkVZrY/8A1gRTzt8yh3vxW4OelrjCLJe/9Hd78p7Qt3wDkP\nged9vFbhX4g+rKoT3YWe812vZxv9DL3d0Hn+oxmz/NH+gMwsSaNVcSTRPOkziWajnBS/9reTVzNc\nQG+33jz/GwKr07Ds0T44LMrg+P2E5ZxFlOXz+Xgmy38B89z9S4lrmt1zZnYp0WypSiqCNOkz2nLO\nQy7n/f8FPkTNt5xWn/OdqGembJrZMWb2qJk9blHWxAfjXUk/+V+VsI38erlZXyfN19Sl7v4sMDnu\nZSZeZFSPbbwfQdIZKPub2ePATfHvYK94V9Le7qsSdwGZUx1XCfn9pXnvX3L35wHcfTHp898Ar0y5\nTFuHJ4BniRYqzSbdwj7ovHMekr/3j7v7Y+6+rrIFlDmu9FJPP7S3W5uwLe08/7yl+cNZYWYHE61y\nPJooZXBiOQwPhfZ2qxN37Uz6ef55a/je28aEawPxVOGFRHlwUjU+IcND7v4FM5vNqxPGpdFp5zwk\nP+9Xm9l84F42fstJk+hu3OqZnj6BvV13Pwc4nugP4CR3/1d45SYdqaXtLQc6EngKOI2osT42Q/ye\nRMMqhxOnEkjxgRna2z0B+CkwiWie/wkQXYxN+gIZe8ohPN4uJ+pkPEvUaP4k5etUPjDfQXQfgK8A\nJPnANLPvEOXPXwD8lgTXQqrlfc5XacU5fx1R/p1H2Pi7EHqrpx/U2wVw93uJeg7VEk1bzOtiapaZ\nN+7+Eht7x59NU15sqbs/a2aT3f1WM/tiwrrm0tt19zLRt4NaSaYtBl9IzXItopJwLQev+sA0szQf\nmDsSJTv7FlE+mp+nLTzwnG/GxdRE532O7/+400uN/pHAm4l6u6eSvrc7mqSNb6bhpSbOvEkj6wdm\nvfTCeQ4RJHnvMw8tNXHmTUM5fWC+5O5lM9vU3ZeaWZq03GNJes5nvpiq2TfN0zONfg693dEkHWPM\n1FumM2beZPrAbEFvK8l7H9JTbufMmzw+MH9v0c3d/2xmV5Hf33vSc/5xd38sYxmafdMkPdPod4Cs\nveWsHxa5aeIHZtPk2FPO+oERJI8PTHc/3cwmE60m3h9IkmMqTyEXU0M+MGQMavQzyjC2nnV4Kfha\nxHiRcmw9c085r2sR7VazuOozwFRaNM8+dl1AGZp90yRq9BMKHVsP6C0361pE18gyth7YU272tYhW\nCVpcFTquHvg7CPnAkDEUyuWmrqQfN8zsd0RjjK+Mrbv7we2tVW8ws7uAA6rH1t19Xrvr1enM7Lfu\nvnPV40Vp8taY2YNEKbFfGVfXIqfup55+cm0fW+9hbRtb73Khi6s0rj4OqdFPTmPrLTZextbbxcNv\nHKRx9XFIjX5yPT+23gbjZWy9bUIWV6Fx9XFJY/oiPcbMbnH397S7HtIevZR7R0Qi6un1MDX6IiI9\nRI2+SO/p+VsG9jJdyBUZp5S0TOrRhVyRcUqLq6Qe9fRFxi8trpLXUKMvMn5pcZW8hhp9kfFLi6vk\nNTSmLyLSQzRlU0Skh6jRFxHpIWr0RUR6iBp9EZEeokZfRKSH/H82vzLbQPrxiQAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train = annotated_data.loc[(annotated_data['id'].str.contains('test'))] # swap later\n",
    "valid = annotated_data.loc[(annotated_data['id'].str.contains('validation'))]\n",
    "test = annotated_data.loc[(annotated_data['id'].str.contains('development'))] # swap later\n",
    "\n",
    "train_valid = pd.concat([train, valid, test])\n",
    "\n",
    "answer_columns = ['label']\n",
    "excl = ['id']\n",
    "excl += ['a_out', 'b_out']\n",
    "feature_columns = [col for col in annotated_data.columns if col not in answer_columns and col not in excl]\n",
    "\n",
    "meta_features_train_valid = train_valid[feature_columns].fillna(0)\n",
    "meta_features_test = test[feature_columns].fillna(0)\n",
    "\n",
    "sns.heatmap(meta_features_train_valid.corr())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n",
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n",
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n",
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n",
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "for col in meta_features_train_valid.columns:\n",
    "    std = MinMaxScaler()\n",
    "    meta_features_train_valid[col] = std.fit_transform(meta_features_train_valid[col].values.reshape(-1, 1))\n",
    "    meta_features_test[col] = std.transform(meta_features_test[col].values.reshape(-1, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Keep all BERTs learn on the same corpus\n",
    "remove_test = []\n",
    "remove_validation = []\n",
    "remove_development = [209, 1506, 1988]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "meta_features_train_valid = meta_features_train_valid.values\n",
    "meta_features_test = meta_features_test.values # These should be fix on the stage 2 because they will use to train\n",
    "meta_features_test = np.delete(meta_features_test, remove_development, 0)\n",
    "\n",
    "meta_features_train_valid = np.delete(meta_features_train_valid, remove_test, 0)\n",
    "meta_features_train_valid = np.delete(meta_features_train_valid, [v + len(train) for v in remove_validation], 0)\n",
    "meta_features_train_valid = np.delete(meta_features_train_valid, [v + len(train) + len(valid) for v in remove_development], 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "meta_features_train_valid shpae: (4451, 17)\n",
      "meta_features_test shpae: (1997, 17)\n"
     ]
    }
   ],
   "source": [
    "print(\"meta_features_train_valid shpae:\", meta_features_train_valid.shape)\n",
    "print(\"meta_features_test shpae:\", meta_features_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLI Acrhitecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_mlp_model(input_shape, split_size, meta_features_shape,):\n",
    "    X_input = layers.Input(input_shape)\n",
    "    \n",
    "    meta_input = layers.Input(meta_features_shape)\n",
    "    meta_out = layers.Dropout(0.1, seed = 7)(meta_input)\n",
    "    meta_out = layers.Dense(20, activation='selu')(meta_out)\n",
    "    \n",
    "    X = layers.Dropout(dropout_rate, seed = 7)(X_input)\n",
    "    X = layers.Dense(dense_layer_sizes[0], name = 'dense0')(X)\n",
    "    X = layers.Concatenate()([meta_out, X])\n",
    "    \n",
    "    X = layers.BatchNormalization(name = 'bn0')(X)\n",
    "    X = layers.Activation('relu')(X)\n",
    "    X = layers.Dropout(dropout_rate, seed = 7)(X)\n",
    "\n",
    "    # Output layer\n",
    "    X = layers.Dense(3, name = 'output', kernel_regularizer = regularizers.l2(lambd))(X)\n",
    "    X = layers.Activation('softmax')(X)\n",
    "\n",
    "    # Create model\n",
    "    model = models.Model(input = [X_input, meta_input], output = X, name = \"classif_model\")\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dev_filename = \"contextual_embeddings_gap_development.json\"\n",
    "val_filename = \"contextual_embeddings_gap_validation.json\"\n",
    "test_filename = \"contextual_embeddings_gap_test.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parse_json(embeddings, embedding_size):\n",
    "    '''\n",
    "    Parses the embeddigns given by BERT, and suitably formats them to be passed to the MLP model\n",
    "\n",
    "    Input: embeddings, a DataFrame containing contextual embeddings from BERT, as well as the labels for the classification problem\n",
    "    columns: \"emb_A\": contextual embedding for the word A\n",
    "             \"emb_B\": contextual embedding for the word B\n",
    "             \"emb_P\": contextual embedding for the pronoun\n",
    "             \"label\": the answer to the coreference problem: \"A\", \"B\" or \"NEITHER\"\n",
    "\n",
    "    Output: X, a numpy array containing, for each line in the GAP file, the concatenation of the embeddings of the target words\n",
    "            Y, a numpy array containing, for each line in the GAP file, the one-hot encoded answer to the coreference problem\n",
    "    '''\n",
    "    embeddings.sort_index(inplace = True) # Sorting the DataFrame, because reading from the json file messed with the order\n",
    "    X = np.zeros((len(embeddings),3* embedding_size))\n",
    "    Y = np.zeros((len(embeddings), 3))\n",
    "\n",
    "    # Concatenate features\n",
    "    for i in range(len(embeddings)):\n",
    "        A = np.array(embeddings.loc[i,\"emb_A\"])\n",
    "        B = np.array(embeddings.loc[i,\"emb_B\"])\n",
    "        P = np.array(embeddings.loc[i,\"emb_P\"])\n",
    "        X[i] = np.concatenate((A,B,P))\n",
    "\n",
    "    # One-hot encoding for labels\n",
    "    for i in range(len(embeddings)):\n",
    "        label = embeddings.loc[i,\"label\"]\n",
    "        if label == \"A\":\n",
    "            Y[i,0] = 1\n",
    "        elif label == \"B\":\n",
    "            Y[i,1] = 1\n",
    "        else:\n",
    "            Y[i,2] = 1\n",
    "\n",
    "    return X, Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_folds(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold, model_tag,\n",
    "                dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "                oof_folder='oof/', pred_folder='outputs/'):\n",
    "    '''\n",
    "    Arguments:\n",
    "        tag (data_tag): the tag of model and vec\n",
    "        embedding_size: the size of bert embedding\n",
    "        layer: to indicate which layer of bert is going to use\n",
    "        checkpoint_path: the path to model_checkpoint folder\n",
    "        n_fold: the number of CV folds\n",
    "        model_tag: the prefix of the prediction and oof file and model chekpoint\n",
    "    '''\n",
    "\n",
    "    tag = tag + str(layer) # follow the original naming style\n",
    "    dev_filename = tag + dev_filename\n",
    "    val_filename = tag + val_filename\n",
    "    test_filename = tag + test_filename\n",
    "    \n",
    "    development = pd.read_json(os.path.join(dev_folder_path, dev_filename))\n",
    "    X_development, Y_development = parse_json(development, embedding_size)\n",
    "\n",
    "    validation = pd.read_json(os.path.join(val_folder_path, val_filename))\n",
    "    X_validation, Y_validation = parse_json(validation, embedding_size)\n",
    "\n",
    "    test = pd.read_json(os.path.join(test_folder_path, test_filename))\n",
    "    X_test, Y_test = parse_json(test, embedding_size)\n",
    "\n",
    "    # There may be a few NaN values, where the offset of a target word is greater than the max_seq_length of BERT.\n",
    "    # They are very few, so I'm just dropping the rows.\n",
    "    # remove_test = [row for row in range(len(X_test)) if np.sum(np.isnan(X_test[row].reshape(-1)))]\n",
    "    X_test = np.delete(X_test, remove_test, 0)\n",
    "    Y_test = np.delete(Y_test, remove_test, 0)\n",
    "\n",
    "    # remove_validation = [row for row in range(len(X_validation)) if np.sum(np.isnan(X_validation[row].reshape(-1)))]\n",
    "    X_validation = np.delete(X_validation, remove_validation, 0)\n",
    "    Y_validation = np.delete(Y_validation, remove_validation, 0)\n",
    "\n",
    "    # We want predictions for all development rows. So instead of removing rows, make them 0\n",
    "    # remove_development = [row for row in range(len(X_development)) if np.sum(np.isnan(X_development[row].reshape(-1)))]\n",
    "    X_development = np.delete(X_development, remove_development, 0)\n",
    "    Y_development = np.delete(Y_development, remove_development, 0)\n",
    "\n",
    "    # Will train on data from the gap-test and gap-validation files, in total 2454 rows\n",
    "    X_train = np.concatenate((X_test, X_validation, X_development), axis = 0)\n",
    "    Y_train = np.concatenate((Y_test, Y_validation, Y_development), axis = 0)\n",
    "\n",
    "    # Will predict probabilities for data from the gap-development file; initializing the predictions\n",
    "    prediction = np.zeros((len(X_development), 3)) # testing predictions\n",
    "\n",
    "    # Training and cross-validation\n",
    "    folds = KFold(n_splits=n_fold, shuffle=True, random_state=3)\n",
    "    scores = []\n",
    "    oof = np.zeros_like(Y_train)\n",
    "    \n",
    "    print(\"X_train shape:\", X_train.shape)\n",
    "    \n",
    "    for fold_n, (train_index, valid_index) in enumerate(folds.split(X_train)):\n",
    "        # split training and validation data\n",
    "        print('Fold', fold_n, 'started at', time.ctime())\n",
    "        X_tr, X_val = X_train[train_index], X_train[valid_index]\n",
    "        meta_X_tr, meta_X_val = meta_features_train_valid[train_index], meta_features_train_valid[valid_index]\n",
    "        \n",
    "        Y_tr, Y_val = Y_train[train_index], Y_train[valid_index]\n",
    "        # Define the model, re-initializing for each fold\n",
    "        classif_model = build_mlp_model([X_train.shape[-1]], split_size=embedding_size, meta_features_shape=[meta_X_tr.shape[-1]])\n",
    "        classif_model.compile(optimizer=optimizers.Adam(lr=learning_rate), loss=\"categorical_crossentropy\")\n",
    "        \n",
    "        callbacks = [kc.EarlyStopping(monitor='val_loss', patience=30, restore_best_weights=True),\n",
    "                     kc.ModelCheckpoint(os.path.join(checkpoint_path, model_tag + tag + str(fold_n) + '.pt'), monitor='val_loss', verbose=0, save_best_only=True, mode='min')]\n",
    "\n",
    "        # train the model\n",
    "        classif_model.fit(x=[X_tr, meta_X_tr], y=Y_tr, epochs=epochs, batch_size=batch_size, \n",
    "                          callbacks=callbacks, validation_data=([X_val, meta_X_val], Y_val), verbose=0)\n",
    "\n",
    "        # make predictions on validation and test data\n",
    "        pred_valid = classif_model.predict(x=[X_val, meta_X_val], verbose=0)\n",
    "        oof[valid_index] = pred_valid\n",
    "        pred = classif_model.predict(x=[X_development, meta_features_test], verbose=0)\n",
    "\n",
    "        # oof[valid_index] = pred_valid.reshape(-1,)\n",
    "        scores.append(log_loss(Y_val, pred_valid))\n",
    "        prediction += pred\n",
    "    \n",
    "    prediction /= n_fold\n",
    "\n",
    "    print(\"-\" * 30)\n",
    "    print(\"For the model\", tag)\n",
    "    print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))\n",
    "    print(scores)\n",
    "    print(\"-\" * 30)\n",
    "    \n",
    "    # Write the prediction to file for submission\n",
    "    oof_df = pd.DataFrame(oof)\n",
    "    oof_df.to_csv(oof_folder + model_tag + tag + \".csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bert Base\n",
    "## Uncased"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_base\"\n",
    "embedding_size = 768\n",
    "layer = 8\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-base-uncased-seq512-\"\n",
    "pred_tag = \"Hnaive-\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train shape: (4451, 2304)\n",
      "Fold 0 started at Mon Apr 15 17:04:54 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1 started at Mon Apr 15 17:06:05 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 2 started at Mon Apr 15 17:07:25 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 3 started at Mon Apr 15 17:08:55 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4 started at Mon Apr 15 17:10:22 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 5 started at Mon Apr 15 17:12:01 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 6 started at Mon Apr 15 17:14:15 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------\n",
      "For the model bert-base-uncased-seq512-8\n",
      "CV mean score: 0.4216, std: 0.0216.\n",
      "[0.45284745506352875, 0.4244196294278155, 0.41230956601176105, 0.40155998140208515, 0.38556148088912395, 0.43687804459196344, 0.4376300229289588]\n",
      "------------------------------\n"
     ]
    }
   ],
   "source": [
    "train_folds(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cased"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_base_cased\"\n",
    "embedding_size = 768\n",
    "layer = 8\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-base-cased-seq512-\"\n",
    "pred_tag = \"Hnaive-\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train shape: (4451, 2304)\n",
      "Fold 0 started at Mon Apr 15 20:43:25 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1 started at Mon Apr 15 20:45:06 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 2 started at Mon Apr 15 20:46:40 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 3 started at Mon Apr 15 20:48:44 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4 started at Mon Apr 15 20:50:15 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 5 started at Mon Apr 15 20:52:01 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 6 started at Mon Apr 15 20:53:45 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------\n",
      "For the model bert-base-cased-seq512-8\n",
      "CV mean score: 0.4608, std: 0.0269.\n",
      "[0.4743319799694428, 0.4373354378012492, 0.44027954325113416, 0.4546213620136391, 0.43290349623168606, 0.4702441569633431, 0.5157050030233565]\n",
      "------------------------------\n"
     ]
    }
   ],
   "source": [
    "train_folds(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bert Big\n",
    "## Uncased"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_big\"\n",
    "embedding_size = 1024\n",
    "layer = 19\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-large-uncased-seq300-\"\n",
    "pred_tag = \"Hnaive-\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train shape: (4451, 3072)\n",
      "Fold 0 started at Mon Apr 15 20:55:06 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1 started at Mon Apr 15 20:56:36 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 2 started at Mon Apr 15 20:58:22 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 3 started at Mon Apr 15 20:59:50 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4 started at Mon Apr 15 21:02:10 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 5 started at Mon Apr 15 21:03:43 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 6 started at Mon Apr 15 21:05:19 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------\n",
      "For the model bert-large-uncased-seq300-19\n",
      "CV mean score: 0.3602, std: 0.0292.\n",
      "[0.3598166811612862, 0.3607596384482212, 0.3582503287204572, 0.3308591856716814, 0.31439806618778765, 0.3937398723295094, 0.4033582504961537]\n",
      "------------------------------\n"
     ]
    }
   ],
   "source": [
    "train_folds(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cased"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_big_cased\"\n",
    "embedding_size = 1024\n",
    "layer = 18\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-large-cased-seq300-\"\n",
    "pred_tag = \"Hnaive-\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train shape: (4451, 3072)\n",
      "Fold 0 started at Mon Apr 15 21:06:48 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1 started at Mon Apr 15 21:09:21 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 2 started at Mon Apr 15 21:11:52 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 3 started at Mon Apr 15 21:13:37 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 4 started at Mon Apr 15 21:15:28 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 5 started at Mon Apr 15 21:17:07 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 6 started at Mon Apr 15 21:18:55 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------\n",
      "For the model bert-large-cased-seq300-18\n",
      "CV mean score: 0.3938, std: 0.0245.\n",
      "[0.4055701713716544, 0.3810715370208235, 0.3899906492718075, 0.38371352072414916, 0.3496294121958375, 0.41611255406470327, 0.4305624832985391]\n",
      "------------------------------\n"
     ]
    }
   ],
   "source": [
    "train_folds(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='outputs/', model_tag=pred_tag)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}