model/model_3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import time\n",
    "import numpy as np \n",
    "import pandas as pd \n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()\n",
    "import math\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold\n",
    "from sklearn import metrics\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from keras.layers import Dense, Input, LSTM, CuDNNLSTM, Embedding, CuDNNGRU, MaxPool2D, Conv2D, Concatenate, SpatialDropout1D\n",
    "from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\n",
    "from keras.layers import Reshape, Flatten, Dropout, Activation\n",
    "from keras.layers import Activation, Wrapper\n",
    "from keras.engine.topology import Layer\n",
    "from keras import backend as K\n",
    "\n",
    "from keras.models import Model\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train shape :  (1306122, 3)\n",
      "test shape :  (375806, 2)\n"
     ]
    }
   ],
   "source": [
    "train = pd.read_csv(\"../input/train.csv\")\n",
    "test = pd.read_csv(\"../input/test.csv\")\n",
    "print(\"train shape : \", train.shape)\n",
    "print(\"test shape : \", test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid</th>\n",
       "      <th>question_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000163e3ea7c7a74cd7</td>\n",
       "      <td>Why do so many women become so rude and arroga...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00002bd4fb5d505b9161</td>\n",
       "      <td>When should I apply for RV college of engineer...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00007756b4a147d2b0b3</td>\n",
       "      <td>What is it really like to be a nurse practitio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000086e4b7e1c7146103</td>\n",
       "      <td>Who are entrepreneurs?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0000c4c3fbe8785a3090</td>\n",
       "      <td>Is education really making good people nowadays?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    qid                                      question_text\n",
       "0  0000163e3ea7c7a74cd7  Why do so many women become so rude and arroga...\n",
       "1  00002bd4fb5d505b9161  When should I apply for RV college of engineer...\n",
       "2  00007756b4a147d2b0b3  What is it really like to be a nurse practitio...\n",
       "3  000086e4b7e1c7146103                             Who are entrepreneurs?\n",
       "4  0000c4c3fbe8785a3090   Is education really making good people nowadays?"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load and create embedding matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_word_embedding(filepath):\n",
    "    \"\"\"\n",
    "    Given a filepath to embeddings file, return a word to vec dictionary, \n",
    "    in other words, word_embedding\n",
    "    Ex. {'word': array([1.97, -0.63, ..., 0.573, 2.54])}\n",
    "    \"\"\"\n",
    "    def _get_vec(word, *arr):\n",
    "        return word, np.asarray(arr, dtype='float32')\n",
    "\n",
    "    print('load word embedding ......')\n",
    "    \n",
    "    try:\n",
    "        word_embedding = dict(_get_vec(*w.split(' ')) for w in open(filepath))\n",
    "    except UnicodeDecodeError:\n",
    "        word_embedding = dict(_get_vec(*w.split(' ')) for w in open(\n",
    "            filepath, encoding=\"utf8\", errors='ignore'))\n",
    "    \n",
    "    # sanity check for word vectors must with lengths of 300\n",
    "    words_to_del = []\n",
    "    for word, vec in word_embedding.items():\n",
    "        if len(vec) != 300:\n",
    "            words_to_del.append(word)\n",
    "    for word in words_to_del:\n",
    "        del word_embedding[word]\n",
    "    return word_embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load word embedding ......\n"
     ]
    }
   ],
   "source": [
    "EMBEDDING_FILE_glove = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n",
    "embeddings_index_glove = load_word_embedding(EMBEDDING_FILE_glove)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "regular_punct = list(string.punctuation)\n",
    "extra_punct = [\n",
    "        ',', '.', '\"', ':', ')', '(', '!', '?', '|', ';', \"'\", '$', '&',\n",
    "        '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£',\n",
    "        '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',\n",
    "        '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '“', '★', '”',\n",
    "        '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾',\n",
    "        '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼', '⊕', '▼',\n",
    "        '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',\n",
    "        'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»',\n",
    "        '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø',\n",
    "        '¹', '≤', '‡', '√', '«', '»', '´', 'º', '¾', '¡', '§', '£', '₤']\n",
    "\n",
    "all_punct = list(set(regular_punct + extra_punct))\n",
    "\n",
    "def spacing_punctuation(text):\n",
    "    \"\"\"\n",
    "    add space before and after punctuation and symbols\n",
    "    \"\"\"\n",
    "    for punc in all_punct:\n",
    "        if punc in text:            \n",
    "            text = text.replace(punc, f\" {punc} \")\n",
    "            \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_misspell(text):\n",
    "    \"\"\"\n",
    "    misspell list (quora vs. glove)\n",
    "    \"\"\"\n",
    "    \n",
    "    mispell_dict = {'demonitisation': 'demonetization', 'demonitization': 'demonetization', \n",
    "                    'demonetisation': 'demonetization',\n",
    "                    'pokémon': 'pokemon',\n",
    "                    'Wjy': 'Why',\n",
    "                    'Whst' : 'What',\n",
    "                    'BNBR': 'Be Nice Be Respectful',\n",
    "                    'Bolsonaro': 'Jair Bolsonaro',\n",
    "                    'XXXTentacion': 'Tentacion',\n",
    "                    'Žižek': 'Slovenian philosopher Slavoj Žižek',\n",
    "                    'Adityanath': 'Indian monk Yogi Adityanath',\n",
    "                    'Brexit': 'British Exit',\n",
    "                    'Brexiter': 'British Exit supporter',\n",
    "                    'Brexiters': 'British Exit supporters',\n",
    "                    'Brexiteer': 'British Exit supporter',\n",
    "                    'Brexiteers': 'British Exit supporters',\n",
    "                    'Brexiting': 'British Exit',\n",
    "                    'Brexitosis': 'British Exit disorder',\n",
    "                    'brexit': 'British Exit',\n",
    "                    'brexiters': 'British Exit supporters',\n",
    "                    'cryptocurrencies': 'cryptocurrency',\n",
    "                    'Cryptocurrency': 'cryptocurrency',\n",
    "                    'Litecoin' : 'cryptocurrency',\n",
    "                    'litecoin' : 'cryptocurrency',\n",
    "                    'altcoin' : 'cryptocurrency',\n",
    "                    'altcoins' : 'cryptocurrency',\n",
    "                    'jallikattu': 'Jallikattu',\n",
    "                    'Swachh': 'Swachh Bharat mission campaign ',\n",
    "                    'SJWs': 'social justice warrior',\n",
    "                    'Quorans': 'Quoran',\n",
    "                    'Qoura': 'Quora',\n",
    "                    'quoras': 'Quora',\n",
    "                    'Quroa': 'Quora',\n",
    "                    'QUORA': 'Quora',\n",
    "                    'Qoura': 'Quora',\n",
    "                    'narcissit': 'narcissist',\n",
    "                    'ethereum': 'Ethereum',\n",
    "                    'Blockchain': 'blockchain',\n",
    "                    'UCEED': 'Undergraduate Common Entrance Examination for Design',\n",
    "                    'GDPR': 'General Data Protection Regulation',\n",
    "                    'Redmi' : 'Xiaomi smartphone',\n",
    "                    'OnePlus': 'Android smartphone',\n",
    "                    'Machedo' : 'hot guy',\n",
    "                    'Coinbase':'bitcoin broker',\n",
    "                    'coinbase':'bitcoin broker',\n",
    "                    'DCEU' : 'American media franchise',\n",
    "                    'IIEST': 'Indian Institutes of Engineering Science and Technology',\n",
    "                    'Upwork' : 'global freelancing platform',\n",
    "                    'upwork' : 'global freelancing platform',\n",
    "                    'HackerRank' : 'technology company focuses on competitive programming challenges',\n",
    "                    'pokémon': 'pokemon'}\n",
    "\n",
    "    misspell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n",
    "\n",
    "    def _replace(match):\n",
    "        \"\"\"\n",
    "        reference: https://www.kaggle.com/hengzheng/attention-capsule-why-not-both-lb-0-694 # noqa\n",
    "        \"\"\"\n",
    "        try:\n",
    "            word = mispell_dict.get(match.group(0))\n",
    "        except KeyError:\n",
    "            word = match.group(0)\n",
    "            print('!!Error: Could Not Find Key: {}'.format(word))\n",
    "        return word\n",
    "    return misspell_re.sub(_replace, text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(text):\n",
    "    \"\"\"\n",
    "    preprocess text main steps\n",
    "\n",
    "    \"\"\"\n",
    "    text = spacing_punctuation(text)\n",
    "    text = clean_misspell(text)\n",
    "   \n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "## some config values \n",
    "embed_size = 300 # how big is each word vector\n",
    "max_features = 150000 # how many unique words to use (i.e num rows in embedding vector)\n",
    "maxlen = 75 # max number of words in a question to use"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tokenize text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1306122/1306122 [00:26<00:00, 48668.51it/s]\n",
      "100%|██████████| 375806/375806 [00:07<00:00, 50861.42it/s]\n"
     ]
    }
   ],
   "source": [
    "## preprocess\n",
    "train['question_text'] = train['question_text'].progress_apply(preprocess)\n",
    "test['question_text'] = test['question_text'].progress_apply(preprocess)\n",
    "\n",
    "## split to train and val\n",
    "#train_df, val_df = train_test_split(train, test_size=0.05, random_state=42)\n",
    "\n",
    "## fill up the missing values\n",
    "# train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n",
    "# val_X = val_df[\"question_text\"].fillna(\"_##_\").values\n",
    "\n",
    "train_X = train[\"question_text\"].fillna(\"_##_\").values\n",
    "test_X = test[\"question_text\"].fillna(\"_##_\").values\n",
    "\n",
    "## Tokenize the sentences\n",
    "tokenizer = Tokenizer(num_words=max_features) # glove\n",
    "\n",
    "tokenizer.fit_on_texts(list(train_X))\n",
    "train_X = tokenizer.texts_to_sequences(train_X)\n",
    "#val_X = tokenizer.texts_to_sequences(val_X)\n",
    "test_X = tokenizer.texts_to_sequences(test_X)\n",
    "\n",
    "## Pad the sentences \n",
    "train_X = pad_sequences(train_X, maxlen=maxlen)\n",
    "#val_X = pad_sequences(val_X, maxlen=maxlen)\n",
    "test_X = pad_sequences(test_X, maxlen=maxlen)\n",
    "\n",
    "## Get the target values\n",
    "#train_y = train_df['target'].values\n",
    "train_y = train['target'].values\n",
    "#val_y = val_df['target'].values  \n",
    "\n",
    "#shuffling the data\n",
    "np.random.seed(42)\n",
    "trn_idx = np.random.permutation(len(train_X))\n",
    "#val_idx = np.random.permutation(len(val_X))\n",
    "\n",
    "train_X = train_X[trn_idx]\n",
    "#val_X = val_X[val_idx]\n",
    "train_y = train_y[trn_idx]\n",
    "#val_y = val_y[val_idx] \n",
    "\n",
    "## word index dictionary from input text/document\n",
    "word_index = tokenizer.word_index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## create embedding matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_embedding_weights(word_index, embeddings_index, max_features):\n",
    "    '''\n",
    "    input: word_index from training set and embedding from embedding file\n",
    "    output: matrix with matched embedding arrays\n",
    "    '''\n",
    "    \n",
    "    all_embs = np.stack(embeddings_index.values())\n",
    "    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n",
    "    embed_size = all_embs.shape[1]\n",
    "    nb_words = min(max_features, len(word_index))\n",
    "    \n",
    "    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n",
    "    \n",
    "    # matching the word and updating the corresponding embedding vector  \n",
    "    for word, i in word_index.items():\n",
    "        if i >= max_features: continue\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None: \n",
    "            embedding_matrix[i] = embedding_vector\n",
    "            \n",
    "    return embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:7: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  import sys\n"
     ]
    }
   ],
   "source": [
    "embedding_matrix = create_embedding_weights(word_index, embeddings_index_glove, max_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lstm_model(embedding_matrix):\n",
    "    \n",
    "    input_layer = Input(shape=(maxlen,))\n",
    "    # create embedding layer\n",
    "    x = Embedding(max_features, embed_size, weights = [embedding_matrix])(input_layer)\n",
    "    x = SpatialDropout1D(0.2)(x)\n",
    "    # bidirectional lstm\n",
    "    x1 = Bidirectional(CuDNNLSTM(128, return_sequences=True), name='bidirectional_lstm')(x)\n",
    "    # bidirectional gru\n",
    "    x2 = Bidirectional(CuDNNGRU(128, return_sequences=True), name='bidirectional_gru')(x1)\n",
    "    # attention\n",
    "    #atten = Attention(step_dim=maxlen)(x)\n",
    "    # global_max_pooling1d\n",
    "\n",
    "    max_pool1 = GlobalMaxPooling1D()(x1)\n",
    "    max_pool2 = GlobalMaxPooling1D()(x2)\n",
    "    x = Concatenate()([max_pool1, max_pool2])\n",
    "    #x = Dense(32, activation=\"relu\", name = 'dense_1')(x)\n",
    "    #x = Dense(16, activation=\"relu\", name = 'dense_2')(x)\n",
    "    # output layer\n",
    "    output_layer = Dense(1, activation=\"sigmoid\", name = 'output')(x)\n",
    "    model = Model(inputs=input_layer, outputs=output_layer)\n",
    "    # compile model\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/suicaokhoailang/lstm-attention-baseline-0-652-lb\n",
    "\n",
    "class Attention(Layer):\n",
    "    def __init__(self, step_dim,\n",
    "                 W_regularizer=None, b_regularizer=None,\n",
    "                 W_constraint=None, b_constraint=None,\n",
    "                 bias=True, **kwargs):\n",
    "        self.supports_masking = True\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(W_regularizer)\n",
    "        self.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        self.step_dim = step_dim\n",
    "        self.features_dim = 0\n",
    "        super(Attention, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        self.features_dim = input_shape[-1]\n",
    "        \n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((input_shape[1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "        else:\n",
    "            self.b = None\n",
    "\n",
    "        self.built = True\n",
    "\n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        features_dim = self.features_dim\n",
    "        step_dim = self.step_dim\n",
    "\n",
    "        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n",
    "                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n",
    "        if self.bias:\n",
    "            eij += self.b\n",
    "\n",
    "        eij = K.tanh(eij)\n",
    "\n",
    "        a = K.exp(eij)\n",
    "\n",
    "        if mask is not None:\n",
    "            a *= K.cast(mask, K.floatx())\n",
    "\n",
    "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "\n",
    "        a = K.expand_dims(a)\n",
    "        weighted_input = x * a\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return input_shape[0],  self.features_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/strideradu/word2vec-and-gensim-go-go-go\n",
    "\n",
    "# def train_pred(model, epochs=2):\n",
    "#     for e in range(epochs):\n",
    "#         model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))\n",
    "#         pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)\n",
    "#     pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)\n",
    "    \n",
    "    \n",
    "    # Plot training & validation accuracy values\n",
    "#     plt.plot(model.history['acc'])\n",
    "#     plt.plot(model.history['val_acc'])\n",
    "#     plt.title('Model accuracy')\n",
    "#     plt.ylabel('Accuracy')\n",
    "#     plt.xlabel('Epoch')\n",
    "#     plt.legend(['Train', 'Test'], loc='upper left')\n",
    "#     plt.show()\n",
    "\n",
    "#     # Plot training & validation loss values\n",
    "#     plt.plot(model.history.history['loss'])\n",
    "#     plt.plot(model.history.history['val_loss'])\n",
    "#     plt.title('Model loss')\n",
    "#     plt.ylabel('Loss')\n",
    "#     plt.xlabel('Epoch')\n",
    "#     plt.legend(['Train', 'Test'], loc='upper left')\n",
    "#     plt.show()\n",
    "#     return pred_val_y, pred_test_y\n",
    "\n",
    "\n",
    "\n",
    "def train_pred(model, train_X, train_y, val_X, val_y, input_threshold, epochs=2):\n",
    "    for epo in range(epochs):\n",
    "        # train model\n",
    "        model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y), verbose=0)\n",
    "        # create validation set's predictions\n",
    "        pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)\n",
    "        # check score of validation set\n",
    "        best_score = metrics.f1_score(val_y, (pred_val_y > input_threshold).astype(int))\n",
    "        print(\"Epoch: \", epo, \"-    Val F1 Score: {:.4f}\".format(best_score))\n",
    "    # create test set's predictions \n",
    "    pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)\n",
    "    print('=' * 60)\n",
    "    return pred_val_y, pred_test_y, best_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model = lstm_model(embedding_matrix)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#best_thresh = 0.33\n",
    "#pred_val_y, pred_test_y = train_pred(model, train_X, train_y, val_X, val_y, best_thresh, epochs=4) # GloVe only"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## StratifiedKFold splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "Epoch:  0 -    Val F1 Score: 0.6738\n",
      "Epoch:  1 -    Val F1 Score: 0.6778\n",
      "Epoch:  2 -    Val F1 Score: 0.6769\n",
      "============================================================\n",
      "Epoch:  0 -    Val F1 Score: 0.6728\n",
      "Epoch:  1 -    Val F1 Score: 0.6776\n",
      "Epoch:  2 -    Val F1 Score: 0.6750\n",
      "============================================================\n",
      "Epoch:  0 -    Val F1 Score: 0.6631\n",
      "Epoch:  1 -    Val F1 Score: 0.6790\n",
      "Epoch:  2 -    Val F1 Score: 0.6762\n",
      "============================================================\n",
      "Epoch:  0 -    Val F1 Score: 0.6724\n",
      "Epoch:  1 -    Val F1 Score: 0.6822\n",
      "Epoch:  2 -    Val F1 Score: 0.6789\n",
      "============================================================\n"
     ]
    }
   ],
   "source": [
    "best_thresh = 0.36\n",
    "# placeholder for prediction vector\n",
    "train_meta = np.zeros(train_y.shape[0])\n",
    "test_meta = np.zeros(test_X.shape[0])\n",
    "\n",
    "# create 4 folds, StratifiedKFold b/c imbalanced class\n",
    "folds = StratifiedKFold(n_splits=4, shuffle=True, random_state=42)\n",
    "splits = list(folds.split(X = train_X, y = train_y))\n",
    "\n",
    "for idx, (train_idx, valid_idx) in enumerate(splits):\n",
    "        X_train = train_X[train_idx]\n",
    "        y_train = train_y[train_idx]\n",
    "        X_val = train_X[valid_idx]\n",
    "        y_val = train_y[valid_idx]\n",
    "        \n",
    "        model = lstm_model(embedding_matrix)\n",
    "        # training the model\n",
    "        pred_val_y, pred_test_y, best_score = train_pred(model, X_train, y_train, X_val, y_val, best_thresh, epochs = 3)\n",
    "        # add the predictions to \n",
    "        train_meta[valid_idx] = pred_val_y.reshape(-1)\n",
    "        # add the test set's predictions to test_meta\n",
    "        test_meta += pred_test_y.reshape(-1) / len(splits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# thresholds = []\n",
    "# for thresh in np.arange(0.1, 0.501, 0.01):\n",
    "#     thresh = np.round(thresh, 2)\n",
    "#     res = metrics.f1_score(val_y, (pred_val_y > thresh).astype(int))\n",
    "#     thresholds.append([thresh, res])\n",
    "#     print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n",
    "    \n",
    "# thresholds.sort(key=lambda x: x[1], reverse=True)\n",
    "# best_thresh = thresholds[0][0]\n",
    "# print(\"Best threshold: \", best_thresh)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## the final F1 score of the entire test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6767521978085285"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.f1_score(y_true=train_y, y_pred=train_meta > best_thresh)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pred_test_y = (pred_test_y > best_thresh).astype(int)\n",
    "# test_df = pd.read_csv(\"../input/test.csv\", usecols=[\"qid\"])\n",
    "# out_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\n",
    "# out_df['prediction'] = pred_test_y\n",
    "# out_df.to_csv(\"submission.csv\", index=False)\n",
    "\n",
    "\n",
    "final_test_y = (test_meta > best_thresh).astype(int)\n",
    "test_df = pd.read_csv(\"../input/test.csv\", usecols=[\"qid\"])\n",
    "out_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\n",
    "out_df['prediction'] = final_test_y\n",
    "out_df.to_csv(\"submission.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}