final-sub2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_uuid": "a7fe0e6e0364e1e7e0570b0563dc5be69632430a"
   },
   "outputs": [],
   "source": [
    "FOLD = 0\n",
    "\n",
    "import os\n",
    "import time\n",
    "import math\n",
    "import requests\n",
    "import glob\n",
    "\n",
    "import ast\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import mlcrate as mlc\n",
    "\n",
    "import os\n",
    "\n",
    "import cv2\n",
    "\n",
    "from sklearn.model_selection import train_test_split, KFold, StratifiedKFold\n",
    "from sklearn.metrics import fbeta_score, accuracy_score, precision_score, recall_score, f1_score\n",
    "\n",
    "from skimage.transform import resize\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from PIL import Image, ImageDraw\n",
    "\n",
    "from tqdm import tqdm, tqdm_notebook\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from torch import optim\n",
    "from torch.optim import Optimizer\n",
    "from torch.utils.data import  DataLoader\n",
    "from torchtext.data import Dataset\n",
    "import torch.utils.checkpoint as checkpoint\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize, TweetTokenizer\n",
    "import torchvision\n",
    "from torchvision import transforms, utils\n",
    "import random\n",
    "import torchtext\n",
    "import torchtext.data as data\n",
    "\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "\n",
    "import spacy\n",
    "from spacy.lang.en import English\n",
    "\n",
    "SEED = 1337\n",
    "\n",
    "NOTIFY_EACH_EPOCH = False\n",
    "\n",
    "WORKERS = 0\n",
    "BATCH_SIZE = 512\n",
    "\n",
    "N_SPLITS = 10\n",
    "\n",
    "np.random.seed(SEED)\n",
    "# torch.manual_seed(SEED)\n",
    "# torch.cuda.manual_seed(SEED)\n",
    "# torch.backends.cudnn.deterministic = True\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "def get_n_params(model):\n",
    "    pp=0\n",
    "    for p in list(model.parameters()):\n",
    "        nn=1\n",
    "        for s in list(p.size()):\n",
    "            nn = nn*s\n",
    "        pp += nn\n",
    "    return pp\n",
    "\n",
    "# from https://github.com/floydhub/save-and-resume\n",
    "def save_checkpoint(state):\n",
    "    \"\"\"Save checkpoint if a new best is achieved\"\"\"\n",
    "    print (\" Saving checkpoint\")\n",
    "\n",
    "    filename = f'./checkpoint-{state[\"epoch\"]}.pt.tar'\n",
    "    torch.save(state, filename)\n",
    "    \n",
    "def initialize(model, path=None, optimizer=None):   \n",
    "    if path == None:\n",
    "        checkpoints = glob.glob('./*.pt.tar')\n",
    "        path = checkpoints[np.argmax([int(checkpoint.split('checkpoint-')[1].split('.')[0]) for checkpoint in checkpoints])]\n",
    "\n",
    "    checkpoint = torch.load(path)\n",
    "\n",
    "    model.load_state_dict(checkpoint['model'])\n",
    "\n",
    "    print(f' Loaded checkpoint {path} | Trained for {checkpoint[\"epoch\"] + 1} epochs')\n",
    "    \n",
    "    if optimizer:\n",
    "        optimizer.load_state_dict(checkpoint['optimizer'])\n",
    "          \n",
    "        epoch = checkpoint['epoch'] + 1\n",
    "        train_iteration = checkpoint['train_iteration']\n",
    "        val_iteration = checkpoint['val_iteration']\n",
    "\n",
    "        return model, optimizer, epoch, train_iteration, val_iteration\n",
    "    else:\n",
    "        return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_uuid": "5e44cc342b41cc0c30311923a12f2690330fb733"
   },
   "outputs": [],
   "source": [
    "\n",
    "def seed_everything(seed=1029):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "seed_everything()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_uuid": "6d336dafb832c8c10084b7e4cd5de12282bfba28"
   },
   "outputs": [],
   "source": [
    "\n",
    "puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n",
    " '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n",
    " '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n",
    " '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n",
    " '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n",
    "\n",
    "def clean_text(x):\n",
    "    x = str(x)\n",
    "    for punct in puncts:\n",
    "        if punct in x:\n",
    "            x = x.replace(punct, f' {punct} ')\n",
    "    return x\n",
    "\n",
    "def clean_numbers(x):\n",
    "    x = re.sub('[0-9]{5,}', '#####', x)\n",
    "    x = re.sub('[0-9]{4}', '####', x)\n",
    "    x = re.sub('[0-9]{3}', '###', x)\n",
    "    x = re.sub('[0-9]{2}', '##', x)\n",
    "    return x\n",
    "\n",
    "mispell_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\n",
    "\n",
    "def _get_mispell(mispell_dict):\n",
    "    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n",
    "    return mispell_dict, mispell_re\n",
    "\n",
    "mispellings, mispellings_re = _get_mispell(mispell_dict)\n",
    "def replace_typical_misspell(text):\n",
    "    def replace(match):\n",
    "        return mispellings[match.group(0)]\n",
    "    return mispellings_re.sub(replace, text)\n",
    "\n",
    "\n",
    "def add_features(x):\n",
    "    \n",
    "    df = x.copy()\n",
    "    \n",
    "    df['question_text'] = df['question_text'].apply(lambda x:str(x))\n",
    "    df['total_length'] = df['question_text'].apply(len)\n",
    "    df['capitals'] = df['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))\n",
    "    df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']), axis=1)\n",
    "    df['num_words'] = df.question_text.str.count('\\S+')\n",
    "    df['num_unique_words'] = df['question_text'].apply(lambda comment: len(set(w for w in comment.split())))\n",
    "    df['words_vs_unique'] = df['num_unique_words'] / df['num_words']  \n",
    "    \n",
    "    df = df[['total_length','caps_vs_length','words_vs_unique']].fillna(0)\n",
    "    \n",
    "    return df\n",
    "\n",
    "def load_and_prec(train_df, test_df):\n",
    "    features = add_features(train_df)\n",
    "    test_features = add_features(test_df)\n",
    "    ss = StandardScaler()\n",
    "    ss.fit(np.vstack((features, test_features)))\n",
    "    features = ss.transform(features)\n",
    "    test_features = ss.transform(test_features)\n",
    "    train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: x.lower())\n",
    "    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: x.lower())\n",
    "\n",
    "    # Clean the text\n",
    "    train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: clean_text(x))\n",
    "    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_text(x))\n",
    "    \n",
    "    # Clean numbers\n",
    "    train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: clean_numbers(x))\n",
    "    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_numbers(x))\n",
    "    \n",
    "    # Clean speelings\n",
    "    train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n",
    "    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n",
    "\n",
    "    return train_df, test_df, features, test_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_uuid": "321593e69c05e023da01c4019576ba09e34cafd1"
   },
   "outputs": [],
   "source": [
    "sample_submission = pd.read_csv('../input/sample_submission.csv')\n",
    "train = pd.read_csv('../input/train.csv')\n",
    "test = pd.read_csv('../input/test.csv')\n",
    "all_train_preds = np.zeros((len(train)))\n",
    "real_valid_preds = np.zeros((len(train)))\n",
    "test_preds = np.zeros((len(test)))\n",
    "y_train_label = train.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_uuid": "e4e3d795b5e36576dc98d5319faae7887b4a4d26",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:56: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:57: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n"
     ]
    }
   ],
   "source": [
    "train, test, train_features, test_features = load_and_prec(train, test)\n",
    "enable_local_test = False\n",
    "if enable_local_test:\n",
    "    n_test = len(test) * 4\n",
    "    train, test = (train.iloc[:-n_test].reset_index(drop=True), \n",
    "                               train.iloc[-n_test:].reset_index(drop=True))\n",
    "    test_y = test['target'].values\n",
    "    y_train_label = train.target\n",
    "    test_preds = np.zeros((len(test)))\n",
    "    all_train_preds = np.zeros((len(train)))\n",
    "    real_valid_preds = np.zeros((len(train)))\n",
    "else:\n",
    "    pass\n",
    "train.to_csv('../train.csv', index=False)\n",
    "test.to_csv('../test.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_uuid": "4c7e4854b60b1ff433abe2c2082542f58acb43a5"
   },
   "outputs": [],
   "source": [
    "def get_glove():\n",
    "    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n",
    "    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]\n",
    "    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n",
    "    glo_stoi = {}\n",
    "    for i, j in enumerate(list(embeddings_index.keys())):\n",
    "        glo_stoi[j] = i\n",
    "    glo_vectors = torch.tensor(list(embeddings_index.values()),dtype=torch.float)\n",
    "    return glo_stoi, glo_vectors\n",
    "\n",
    "def get_par():\n",
    "    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n",
    "    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n",
    "    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n",
    "    par_stoi = {}\n",
    "    for i, j in enumerate(list(embeddings_index.keys())):\n",
    "        par_stoi[j] = i\n",
    "    par_vectors = torch.tensor(list(embeddings_index.values()),dtype=torch.float)\n",
    "    return par_stoi, par_vectors\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_uuid": "5170f18e071bbefd214801918b23bf114bb6dcd0"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_uuid": "d52cc97f05e431151f7f488fa39a3ae014610b88",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Dataset: 1306122\n",
      "Test Dataset: 56370\n",
      "CPU times: user 8min 20s, sys: 14.7 s, total: 8min 35s\n",
      "Wall time: 8min 34s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# from http://mlexplained.com/2018/02/08/a-comprehensive-tutorial-to-torchtext\n",
    "question_field = data.Field(lower=True, batch_first=True, include_lengths=True)\n",
    "target_field = data.Field(sequential=False, use_vocab=False, batch_first=True)\n",
    "\n",
    "train_fields = [\n",
    "    ('qid', None),\n",
    "    ('question_text', question_field),\n",
    "    ('target', target_field)\n",
    "]\n",
    "\n",
    "test_fields = [\n",
    "    ('qid', None),\n",
    "    ('question_text', question_field)\n",
    "]\n",
    "\n",
    "train_dataset = data.TabularDataset('../train.csv',  format='CSV', skip_header=True, fields=train_fields)\n",
    "test_dataset = data.TabularDataset('../test.csv', format='CSV', skip_header=True, fields=test_fields)\n",
    "\n",
    "# vectors = torchtext.vocab.Vectors('../input/embeddings/glove.840B.300d/glove.840B.300d.txt', max_vectors=495000)\n",
    "# vectors2 = torchtext.vocab.Vectors('../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt', max_vectors=495000)\n",
    "glo_stoi, glo_vectors = get_glove()\n",
    "question_field.build_vocab(train_dataset, max_size=95000)\n",
    "question_field.vocab.set_vectors(glo_stoi, glo_vectors, 300)\n",
    "glove = question_field.vocab.vectors\n",
    "del glo_stoi, glo_vectors\n",
    "\n",
    "par_stoi, par_vectors = get_par()\n",
    "question_field.vocab.set_vectors(par_stoi, par_vectors, 300)\n",
    "paragram = question_field.vocab.vectors\n",
    "del par_stoi, par_vectors\n",
    "pretrained_embedding = torch.from_numpy(np.mean([glove.cpu().numpy(),paragram.cpu().numpy()], axis=0))\n",
    "\n",
    "\n",
    "\n",
    "print(f'Train Dataset: {len(train_dataset)}')\n",
    "print(f'Test Dataset: {len(test_dataset)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_uuid": "52bf7ebfbd5f0a422a650554635d67864dd79a53"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "408"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del glove, paragram\n",
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_uuid": "50c0cb8013459260616b72d930b12f913c0e823f",
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_uuid": "52ef5b659e612995b903ab699d0ecda3fb077d0f",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import torch as t\n",
    "Routings = 4 #5\n",
    "Num_capsule = 5\n",
    "Dim_capsule = 5#16\n",
    "dropout_p = 0.25\n",
    "rate_drop_dense = 0.28\n",
    "gru_len = 128\n",
    "dropout_p = 0.25\n",
    "rate_drop_dense = 0.28\n",
    "LR = 0.001\n",
    "T_epsilon = 1e-7\n",
    "\n",
    "class CyclicLR(object):\n",
    "    def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,\n",
    "                 step_size=2000, mode='triangular', gamma=1.,\n",
    "                 scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):\n",
    "\n",
    "        if not isinstance(optimizer, Optimizer):\n",
    "            raise TypeError('{} is not an Optimizer'.format(\n",
    "                type(optimizer).__name__))\n",
    "        self.optimizer = optimizer\n",
    "\n",
    "        if isinstance(base_lr, list) or isinstance(base_lr, tuple):\n",
    "            if len(base_lr) != len(optimizer.param_groups):\n",
    "                raise ValueError(\"expected {} base_lr, got {}\".format(\n",
    "                    len(optimizer.param_groups), len(base_lr)))\n",
    "            self.base_lrs = list(base_lr)\n",
    "        else:\n",
    "            self.base_lrs = [base_lr] * len(optimizer.param_groups)\n",
    "\n",
    "        if isinstance(max_lr, list) or isinstance(max_lr, tuple):\n",
    "            if len(max_lr) != len(optimizer.param_groups):\n",
    "                raise ValueError(\"expected {} max_lr, got {}\".format(\n",
    "                    len(optimizer.param_groups), len(max_lr)))\n",
    "            self.max_lrs = list(max_lr)\n",
    "        else:\n",
    "            self.max_lrs = [max_lr] * len(optimizer.param_groups)\n",
    "\n",
    "        self.step_size = step_size\n",
    "\n",
    "        if mode not in ['triangular', 'triangular2', 'exp_range'] \\\n",
    "                and scale_fn is None:\n",
    "            raise ValueError('mode is invalid and scale_fn is None')\n",
    "\n",
    "        self.mode = mode\n",
    "        self.gamma = gamma\n",
    "\n",
    "        if scale_fn is None:\n",
    "            if self.mode == 'triangular':\n",
    "                self.scale_fn = self._triangular_scale_fn\n",
    "                self.scale_mode = 'cycle'\n",
    "            elif self.mode == 'triangular2':\n",
    "                self.scale_fn = self._triangular2_scale_fn\n",
    "                self.scale_mode = 'cycle'\n",
    "            elif self.mode == 'exp_range':\n",
    "                self.scale_fn = self._exp_range_scale_fn\n",
    "                self.scale_mode = 'iterations'\n",
    "        else:\n",
    "            self.scale_fn = scale_fn\n",
    "            self.scale_mode = scale_mode\n",
    "\n",
    "        self.batch_step(last_batch_iteration + 1)\n",
    "        self.last_batch_iteration = last_batch_iteration\n",
    "\n",
    "    def batch_step(self, batch_iteration=None):\n",
    "        if batch_iteration is None:\n",
    "            batch_iteration = self.last_batch_iteration + 1\n",
    "        self.last_batch_iteration = batch_iteration\n",
    "        for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):\n",
    "            param_group['lr'] = lr\n",
    "\n",
    "    def _triangular_scale_fn(self, x):\n",
    "        return 1.\n",
    "\n",
    "    def _triangular2_scale_fn(self, x):\n",
    "        return 1 / (2. ** (x - 1))\n",
    "\n",
    "    def _exp_range_scale_fn(self, x):\n",
    "        return self.gamma**(x)\n",
    "\n",
    "    def get_lr(self):\n",
    "        step_size = float(self.step_size)\n",
    "        cycle = np.floor(1 + self.last_batch_iteration / (2 * step_size))\n",
    "        x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)\n",
    "\n",
    "        lrs = []\n",
    "        param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)\n",
    "        for param_group, base_lr, max_lr in param_lrs:\n",
    "            base_height = (max_lr - base_lr) * np.maximum(0, (1 - x))\n",
    "            if self.scale_mode == 'cycle':\n",
    "                lr = base_lr + base_height * self.scale_fn(cycle)\n",
    "            else:\n",
    "                lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)\n",
    "            lrs.append(lr)\n",
    "        return lrs\n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "class Embed_Layer(nn.Module):\n",
    "    def __init__(self, embedding_matrix=None, vocab_size=None, embedding_dim=300):\n",
    "        super(Embed_Layer, self).__init__()\n",
    "        self.encoder = nn.Embedding(vocab_size + 1, embedding_dim)\n",
    "        if use_pretrained_embedding:\n",
    "            # self.encoder.weight.data.copy_(t.from_numpy(np.load(embedding_path))) # 方法一，加载np.save的npy文件\n",
    "            self.encoder.weight.data.copy_(t.from_numpy(embedding_matrix))  # 方法二\n",
    "\n",
    "    def forward(self, x, dropout_p=0.25):\n",
    "        return nn.Dropout(p=dropout_p)(self.encoder(x))\n",
    "\n",
    "\n",
    "class GRU_Layer(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(GRU_Layer, self).__init__()\n",
    "        self.gru = nn.GRU(input_size=300,\n",
    "                          hidden_size=gru_len,\n",
    "                          bidirectional=True)\n",
    "        '''\n",
    "        自己修改GRU里面的激活函数及加dropout和recurrent_dropout\n",
    "        如果要使用，把rnn_revised import进来，但好像是使用cpu跑的，比较慢\n",
    "       '''\n",
    "        # # if you uncomment /*from rnn_revised import * */, uncomment following code aswell\n",
    "        # self.gru = RNNHardSigmoid('GRU', input_size=300,\n",
    "        #                           hidden_size=gru_len,\n",
    "        #                           bidirectional=True)\n",
    "\n",
    "    # 这步很关键，需要像keras一样用glorot_uniform和orthogonal_uniform初始化参数\n",
    "    def init_weights(self):\n",
    "        ih = (param.data for name, param in self.named_parameters() if 'weight_ih' in name)\n",
    "        hh = (param.data for name, param in self.named_parameters() if 'weight_hh' in name)\n",
    "        b = (param.data for name, param in self.named_parameters() if 'bias' in name)\n",
    "        for k in ih:\n",
    "            nn.init.xavier_uniform_(k)\n",
    "        for k in hh:\n",
    "            nn.init.orthogonal_(k)\n",
    "        for k in b:\n",
    "            nn.init.constant_(k, 0)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.gru(x)\n",
    "\n",
    "\n",
    "# core caps_layer with squash func\n",
    "class Caps_Layer(nn.Module):\n",
    "    def __init__(self, input_dim_capsule=gru_len * 2, num_capsule=Num_capsule, dim_capsule=Dim_capsule, \\\n",
    "                 routings=Routings, kernel_size=(9, 1), share_weights=True,\n",
    "                 activation='default', **kwargs):\n",
    "        super(Caps_Layer, self).__init__(**kwargs)\n",
    "\n",
    "        self.num_capsule = num_capsule\n",
    "        self.dim_capsule = dim_capsule\n",
    "        self.routings = routings\n",
    "        self.kernel_size = kernel_size  # 暂时没用到\n",
    "        self.share_weights = share_weights\n",
    "        if activation == 'default':\n",
    "            self.activation = self.squash\n",
    "        else:\n",
    "            self.activation = nn.ReLU(inplace=True)\n",
    "\n",
    "        if self.share_weights:\n",
    "            self.W = nn.Parameter(\n",
    "                nn.init.xavier_normal_(t.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))\n",
    "        else:\n",
    "            self.W = nn.Parameter(\n",
    "                t.randn(BATCH_SIZE, input_dim_capsule, self.num_capsule * self.dim_capsule))  # 64即batch_size\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        if self.share_weights:\n",
    "            u_hat_vecs = t.matmul(x, self.W)\n",
    "        else:\n",
    "            print('add later')\n",
    "\n",
    "        batch_size = x.size(0)\n",
    "        input_num_capsule = x.size(1)\n",
    "        u_hat_vecs = u_hat_vecs.view((batch_size, input_num_capsule,\n",
    "                                      self.num_capsule, self.dim_capsule))\n",
    "        u_hat_vecs = u_hat_vecs.permute(0, 2, 1, 3)  # 转成(batch_size,num_capsule,input_num_capsule,dim_capsule)\n",
    "        b = t.zeros_like(u_hat_vecs[:, :, :, 0])  # (batch_size,num_capsule,input_num_capsule)\n",
    "\n",
    "        for i in range(self.routings):\n",
    "            b = b.permute(0, 2, 1)\n",
    "            c = F.softmax(b, dim=2)\n",
    "            c = c.permute(0, 2, 1)\n",
    "            b = b.permute(0, 2, 1)\n",
    "            outputs = self.activation(t.einsum('bij,bijk->bik', (c, u_hat_vecs)))  # batch matrix multiplication\n",
    "            # outputs shape (batch_size, num_capsule, dim_capsule)\n",
    "            if i < self.routings - 1:\n",
    "                b = t.einsum('bik,bijk->bij', (outputs, u_hat_vecs))  # batch matrix multiplication\n",
    "        return outputs  # (batch_size, num_capsule, dim_capsule)\n",
    "\n",
    "    # text version of squash, slight different from original one\n",
    "    def squash(self, x, axis=-1):\n",
    "        s_squared_norm = (x ** 2).sum(axis, keepdim=True)\n",
    "        scale = t.sqrt(s_squared_norm + T_epsilon)\n",
    "        return x / scale\n",
    "    \n",
    "class Capsule_Main(nn.Module):\n",
    "    def __init__(self, embedding_matrix=None, vocab_size=None):\n",
    "        super(Capsule_Main, self).__init__()\n",
    "        self.embed_layer = Embed_Layer(embedding_matrix, vocab_size)\n",
    "        self.gru_layer = GRU_Layer()\n",
    "        # 【重要】初始化GRU权重操作，这一步非常关键，acc上升到0.98，如果用默认的uniform初始化则acc一直在0.5左右\n",
    "        self.gru_layer.init_weights()\n",
    "        self.caps_layer = Caps_Layer()\n",
    "        self.dense_layer = Dense_Layer()\n",
    "\n",
    "    def forward(self, content):\n",
    "        content1 = self.embed_layer(content)\n",
    "        content2, _ = self.gru_layer(\n",
    "            content1)  # 这个输出是个tuple，一个output(seq_len, batch_size, num_directions * hidden_size)，一个hn\n",
    "        content3 = self.caps_layer(content2)\n",
    "        output = self.dense_layer(content3)\n",
    "        return output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "_uuid": "01ca8c7d0970d138f4f72ae16b265a11f87cd00f"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "_uuid": "dda7d32355775e045d56ae70f793e55bacbc0651"
   },
   "outputs": [],
   "source": [
    "# from https://discuss.pytorch.org/t/self-attention-on-words-and-masking/5671/4\n",
    "class SelfAttention(nn.Module):\n",
    "    def __init__(self, hidden_size, batch_first=False):\n",
    "        super(SelfAttention, self).__init__()\n",
    "\n",
    "        self.hidden_size = hidden_size\n",
    "        self.batch_first = batch_first\n",
    "\n",
    "        self.att_weights = nn.Parameter(torch.Tensor(1, hidden_size), requires_grad=True)\n",
    "\n",
    "        stdv = 1.0 / math.sqrt(self.hidden_size)\n",
    "        for weight in self.att_weights:\n",
    "            nn.init.uniform_(weight, -stdv, stdv)\n",
    "\n",
    "    def get_mask(self):\n",
    "        pass\n",
    "\n",
    "    def forward(self, inputs, lengths):\n",
    "        if self.batch_first:\n",
    "            batch_size, max_len = inputs.size()[:2]\n",
    "        else:\n",
    "            max_len, batch_size = inputs.size()[:2]\n",
    "            \n",
    "        # apply attention layer\n",
    "        weights = torch.bmm(inputs,\n",
    "                            self.att_weights  # (1, hidden_size)\n",
    "                            .permute(1, 0)  # (hidden_size, 1)\n",
    "                            .unsqueeze(0)  # (1, hidden_size, 1)\n",
    "                            .repeat(batch_size, 1, 1) # (batch_size, hidden_size, 1)\n",
    "                            )\n",
    "    \n",
    "        attentions = torch.softmax(F.relu(weights.squeeze()), dim=-1)\n",
    "\n",
    "        # create mask based on the sentence lengths\n",
    "        mask = torch.ones(attentions.size(), requires_grad=True).cuda()\n",
    "        for i, l in enumerate(lengths):  # skip the first sentence\n",
    "            if l < max_len:\n",
    "                mask[i, l:] = 0\n",
    "\n",
    "        # apply mask and renormalize attention scores (weights)\n",
    "        masked = attentions * mask\n",
    "        _sums = masked.sum(-1).unsqueeze(-1)  # sums per row\n",
    "        \n",
    "        attentions = masked.div(_sums)\n",
    "\n",
    "        # apply attention weights\n",
    "        weighted = torch.mul(inputs, attentions.unsqueeze(-1).expand_as(inputs))\n",
    "\n",
    "        # get the final fixed vector representations of the sentences\n",
    "        representations = weighted.sum(1).squeeze()\n",
    "\n",
    "        return representations, attentions\n",
    "\n",
    "class BaselineLSTM(nn.Module):\n",
    "    def __init__(self, embedding):\n",
    "        super(BaselineLSTM, self).__init__()\n",
    "                \n",
    "        self.embedding = nn.Embedding.from_pretrained(embedding)\n",
    "        self.embedding_dropout = nn.Dropout2d(0.1)\n",
    "        \n",
    "        self.lstm = nn.LSTM(input_size=300, hidden_size=128, num_layers=2, batch_first=True, bidirectional=True)\n",
    "        self.dropout = nn.Dropout(0.1)\n",
    "        self.lincaps = nn.Linear(Num_capsule * Dim_capsule, 1)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.attention = SelfAttention(128*2, batch_first=True)\n",
    "        self.caps_layer = Caps_Layer()\n",
    "        self.fc = nn.Linear(128*2+1, 1)\n",
    "        self.logit = nn.Linear(1, 1)\n",
    "    \n",
    "    @staticmethod\n",
    "    def split_directions(outputs):\n",
    "        direction_size = int(outputs.size(-1) / 2)\n",
    "        forward = outputs[:, :, :direction_size]\n",
    "        backward = outputs[:, :, direction_size:]\n",
    "        return forward, backward\n",
    "    \n",
    "    @staticmethod\n",
    "    def last_by_index(outputs, lengths):\n",
    "        # Index of the last output for each sequence.\n",
    "        idx = (lengths - 1).view(-1, 1).expand(outputs.size(0),\n",
    "                                               outputs.size(2)).unsqueeze(1).cuda()\n",
    "        return outputs.gather(1, idx).squeeze()\n",
    "    \n",
    "    \n",
    "    def last_timestep(self, outputs, lengths, bi=False):\n",
    "        if bi:\n",
    "            forward, backward = self.split_directions(outputs)\n",
    "            last_forward = self.last_by_index(forward, lengths)\n",
    "            last_backward = backward[:, 0, :]\n",
    "            return torch.cat((last_forward, last_backward), dim=-1)\n",
    "\n",
    "        else:\n",
    "            return self.last_by_index(outputs, lengths)\n",
    "        \n",
    "    def forward(self,x, x_len):\n",
    "        x = self.embedding(x)\n",
    "        x = self.embedding_dropout(x)\n",
    "        \n",
    "        x = nn.utils.rnn.pack_padded_sequence(x, x_len, batch_first=True)\n",
    "\n",
    "        out, (hidden, _) = self.lstm(x)\n",
    "        \n",
    "        x, lengths = nn.utils.rnn.pad_packed_sequence(out, batch_first=True)\n",
    "        \n",
    "        \n",
    "        content3 = self.caps_layer(x)\n",
    "        content3 = self.dropout(content3)\n",
    "        batch_size = content3.size(0)\n",
    "        content3 = content3.view(batch_size, -1)\n",
    "        content3 = self.relu(self.lincaps(content3))\n",
    "        \n",
    "        x, _ = self.attention(x, lengths)\n",
    "        x = torch.cat([x, content3,],1)\n",
    "        x = self.fc(x)\n",
    "        x = self.logit(x).view(-1)\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "_uuid": "2a50e01449e45323199c58958e999f8bc3f30579",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "start_epoch = 0\n",
    "epochs = 6\n",
    "early_stopping = 10\n",
    "\n",
    "train_iteration = 0\n",
    "val_iteration = 0\n",
    "\n",
    "threshold = 0.35\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "_uuid": "0cd15c1b6fb9eb5a33ea555e420b0072d7316c0a"
   },
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "_uuid": "afe61e636a4e3adb7d31efa1b3fb76c09c09ea42",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Starting Epoch 0 | LR: 0.001\n",
      " 3m10s | End of Epoch 0 | Train Loss: 0.12371740045207555 | Val Loss: 0.10681805572310306 | Train F1: 0.4808 | Val F1: 0.4775\n",
      " Saving checkpoint\n",
      "\n",
      " Starting Epoch 1 | LR: 0.001\n",
      " 3m09s | End of Epoch 1 | Train Loss: 0.10660146894677196 | Val Loss: 0.10080975141019037 | Train F1: 0.5739 | Val F1: 0.6171\n",
      " Saving checkpoint\n",
      "\n",
      " Starting Epoch 2 | LR: 0.001\n",
      " 3m10s | End of Epoch 2 | Train Loss: 0.10094865910537333 | Val Loss: 0.09940137528612072 | Train F1: 0.6065 | Val F1: 0.5559\n",
      " Saving checkpoint\n",
      "\n",
      " Starting Epoch 3 | LR: 0.001\n",
      " 3m10s | End of Epoch 3 | Train Loss: 0.09587397286371821 | Val Loss: 0.09614817840420803 | Train F1: 0.6306 | Val F1: 0.6402\n",
      " Saving checkpoint\n",
      "\n",
      " Starting Epoch 4 | LR: 0.001\n",
      " 3m10s | End of Epoch 4 | Train Loss: 0.09088067978525617 | Val Loss: 0.09684395751462058 | Train F1: 0.6556 | Val F1: 0.6285\n",
      "\n",
      " Starting Epoch 5 | LR: 0.001\n",
      " 3m10s | End of Epoch 5 | Train Loss: 0.08564619204629029 | Val Loss: 0.09742063982321092 | Train F1: 0.6807 | Val F1: 0.6567\n",
      "\n",
      " Starting Epoch 0 | LR: 0.001\n",
      " 3m14s | End of Epoch 0 | Train Loss: 0.13133982951912093 | Val Loss: 0.10807618285928454 | Train F1: 0.4451 | Val F1: 0.5941\n",
      "\n",
      " Starting Epoch 1 | LR: 0.001\n",
      " 3m10s | End of Epoch 1 | Train Loss: 0.10763019511347481 | Val Loss: 0.10775496017501313 | Train F1: 0.572 | Val F1: 0.4414\n",
      "\n",
      " Starting Epoch 2 | LR: 0.001\n",
      " 3m10s | End of Epoch 2 | Train Loss: 0.10112474077198215 | Val Loss: 0.09820432197703541 | Train F1: 0.6067 | Val F1: 0.6228\n",
      "\n",
      " Starting Epoch 3 | LR: 0.001\n",
      " 3m10s | End of Epoch 3 | Train Loss: 0.09638595376081587 | Val Loss: 0.09747930807143508 | Train F1: 0.6282 | Val F1: 0.6434\n",
      "\n",
      " Starting Epoch 4 | LR: 0.001\n",
      " 3m09s | End of Epoch 4 | Train Loss: 0.09134359024319066 | Val Loss: 0.09708949994959243 | Train F1: 0.6521 | Val F1: 0.6499\n",
      "\n",
      " Starting Epoch 5 | LR: 0.001\n",
      " 3m10s | End of Epoch 5 | Train Loss: 0.0864676146337736 | Val Loss: 0.09860280024547516 | Train F1: 0.6761 | Val F1: 0.6532\n",
      "\n",
      " Starting Epoch 0 | LR: 0.001\n",
      " 3m13s | End of Epoch 0 | Train Loss: 0.12386589663788596 | Val Loss: 0.11450805234013821 | Train F1: 0.4731 | Val F1: 0.3522\n",
      "\n",
      " Starting Epoch 1 | LR: 0.001\n",
      " 3m10s | End of Epoch 1 | Train Loss: 0.10706652750364709 | Val Loss: 0.09965868784869254 | Train F1: 0.5685 | Val F1: 0.6058\n",
      "\n",
      " Starting Epoch 2 | LR: 0.001\n",
      " 3m12s | End of Epoch 2 | Train Loss: 0.1020710590728529 | Val Loss: 0.09828714546683716 | Train F1: 0.5979 | Val F1: 0.6112\n",
      "\n",
      " Starting Epoch 3 | LR: 0.001\n",
      " 3m10s | End of Epoch 3 | Train Loss: 0.09660915404353757 | Val Loss: 0.0970028565791955 | Train F1: 0.6265 | Val F1: 0.6462\n",
      "\n",
      " Starting Epoch 4 | LR: 0.001\n",
      " 3m11s | End of Epoch 4 | Train Loss: 0.09261526485621228 | Val Loss: 0.09737766370060626 | Train F1: 0.6459 | Val F1: 0.6511\n",
      "\n",
      " Starting Epoch 5 | LR: 0.001\n",
      " 3m10s | End of Epoch 5 | Train Loss: 0.08805829465265136 | Val Loss: 0.09706685595476813 | Train F1: 0.6659 | Val F1: 0.635\n",
      "\n",
      " Starting Epoch 0 | LR: 0.001\n",
      " 3m12s | End of Epoch 0 | Train Loss: 0.17722296203901405 | Val Loss: 0.11379811405377378 | Train F1: 0.2679 | Val F1: 0.4478\n",
      "\n",
      " Starting Epoch 1 | LR: 0.001\n",
      " 3m10s | End of Epoch 1 | Train Loss: 0.10948701584524585 | Val Loss: 0.10207001185828342 | Train F1: 0.5529 | Val F1: 0.6174\n",
      "\n",
      " Starting Epoch 2 | LR: 0.001\n",
      " 3m08s | End of Epoch 2 | Train Loss: 0.10231424483166263 | Val Loss: 0.09976622619376374 | Train F1: 0.5938 | Val F1: 0.6186\n",
      "\n",
      " Starting Epoch 3 | LR: 0.001\n",
      " 3m09s | End of Epoch 3 | Train Loss: 0.09680310221519439 | Val Loss: 0.09871552514965753 | Train F1: 0.6227 | Val F1: 0.6571\n",
      "\n",
      " Starting Epoch 4 | LR: 0.001\n",
      " 3m10s | End of Epoch 4 | Train Loss: 0.09239123734001696 | Val Loss: 0.09764123323816376 | Train F1: 0.6451 | Val F1: 0.6184\n",
      "\n",
      " Starting Epoch 5 | LR: 0.001\n",
      " 3m09s | End of Epoch 5 | Train Loss: 0.08792807921102852 | Val Loss: 0.09993935516741995 | Train F1: 0.6654 | Val F1: 0.649\n",
      "\n",
      " Starting Epoch 0 | LR: 0.001\n",
      " 3m14s | End of Epoch 0 | Train Loss: 0.12335775210351001 | Val Loss: 0.10707352635561837 | Train F1: 0.4826 | Val F1: 0.5164\n",
      "\n",
      " Starting Epoch 1 | LR: 0.001\n",
      " 3m11s | End of Epoch 1 | Train Loss: 0.1065511857467443 | Val Loss: 0.10391598041743448 | Train F1: 0.5768 | Val F1: 0.6431\n",
      "\n",
      " Starting Epoch 2 | LR: 0.001\n",
      " 3m10s | End of Epoch 2 | Train Loss: 0.10129052074877057 | Val Loss: 0.09959877499264397 | Train F1: 0.604 | Val F1: 0.6211\n",
      "\n",
      " Starting Epoch 3 | LR: 0.001\n",
      " 3m10s | End of Epoch 3 | Train Loss: 0.09619933055300359 | Val Loss: 0.09741624199772535 | Train F1: 0.631 | Val F1: 0.632\n",
      "\n",
      " Starting Epoch 4 | LR: 0.001\n",
      " 3m10s | End of Epoch 4 | Train Loss: 0.09148285095066577 | Val Loss: 0.09741241884383203 | Train F1: 0.6532 | Val F1: 0.6196\n",
      "\n",
      " Starting Epoch 5 | LR: 0.001\n",
      " 3m10s | End of Epoch 5 | Train Loss: 0.08652069033586704 | Val Loss: 0.0979955494549643 | Train F1: 0.6769 | Val F1: 0.632\n"
     ]
    }
   ],
   "source": [
    "seed_everything()\n",
    "weight = [3, 3, 3, 3, 1]\n",
    "best_train_loss = 1e10\n",
    "best_val_loss = 1e10\n",
    "\n",
    "best_train_f1 = 0\n",
    "best_val_f1 = 0\n",
    "\n",
    "best_epoch = 0\n",
    "\n",
    "timer = mlc.time.Timer()\n",
    "logger = mlc.LinewiseCSVWriter('train_log.csv', header=['epoch', 'lr', 'train_loss', 'val_loss', 'train_f1', 'val_f1'])\n",
    "splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED).split(train, y_train_label))\n",
    "test_preds_local = np.zeros((len(test), len(splits)))\n",
    "for num_fold, (train_idx, val_idx) in enumerate(splits):\n",
    "    \n",
    "    model = BaselineLSTM(pretrained_embedding)\n",
    "    model.cuda()\n",
    "    optimizer = optim.Adam(model.parameters())\n",
    "    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=10, verbose=True)\n",
    "\n",
    "    criterion = nn.BCEWithLogitsLoss()\n",
    "    get_n_params(model)\n",
    "    exasample =  np.asarray(train_dataset.examples)\n",
    "    folds_train_exm = Dataset(fields=train_fields, examples = exasample[train_idx])\n",
    "    folds_val_exm = Dataset(fields=train_fields, examples = exasample[val_idx])\n",
    "    train_dataloader, val_dataloader = data.BucketIterator.splits((folds_train_exm, folds_val_exm), (BATCH_SIZE, BATCH_SIZE), sort_key=lambda x: len(x.question_text), sort_within_batch=True, device=torch.device('cuda'))\n",
    "    test_dataloader = data.BucketIterator(test_dataset, 512, sort=False, shuffle=False, device=torch.device('cuda'))\n",
    "    for epoch in range(start_epoch, epochs):\n",
    "        print(f'\\n Starting Epoch {epoch} | LR: {optimizer.param_groups[0][\"lr\"]}')\n",
    "\n",
    "        train_loss = 0\n",
    "        val_loss = 0\n",
    "\n",
    "        y_train = []\n",
    "        train_preds = []\n",
    "\n",
    "        timer.add(epoch)\n",
    "\n",
    "        model.train()\n",
    "        for i, batch in enumerate(train_dataloader):\n",
    "            (question, length), label = batch.question_text, batch.target.float()\n",
    "            question = question\n",
    "\n",
    "            out = model(question, length)\n",
    "\n",
    "            loss = criterion(out, label)\n",
    "\n",
    "            train_loss += loss.item()\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            loss.backward()\n",
    "            torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0)\n",
    "\n",
    "            optimizer.step()\n",
    "\n",
    "            y_train.append(label.detach())\n",
    "            train_preds.append(out.detach())\n",
    "\n",
    "            train_iteration += 1\n",
    "\n",
    "        model.eval()\n",
    "        valid_preds_fold = np.zeros((len(val_idx)))\n",
    "        with torch.no_grad():\n",
    "\n",
    "            y_val = []\n",
    "            val_preds = []\n",
    "\n",
    "            for j, batch in enumerate(val_dataloader):\n",
    "                (question, length), label = batch.question_text, batch.target.float()\n",
    "                question = question\n",
    "\n",
    "                out = model(question, length)\n",
    "\n",
    "                loss = criterion(out, label)\n",
    "\n",
    "                val_loss += loss.item()\n",
    "\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                y_val.append(label.detach())\n",
    "                val_preds.append(out.detach())\n",
    "    \n",
    "                real_pred_y = sigmoid(out.detach().cpu().numpy())\n",
    "                valid_preds_fold[j * BATCH_SIZE:(j+1) * BATCH_SIZE] = real_pred_y\n",
    "                val_iteration += 1\n",
    "\n",
    "        train_loss /= (i + 1)\n",
    "        val_loss /= (j + 1)\n",
    "\n",
    "        y_train = torch.cat(y_train, dim=0).reshape(-1, 1)\n",
    "        y_val = torch.cat(y_val, dim=0).reshape(-1, 1)\n",
    "\n",
    "        train_preds = torch.cat(train_preds, dim=0).reshape(-1, 1)\n",
    "        val_preds = torch.cat(val_preds, dim=0).reshape(-1, 1)\n",
    "\n",
    "        train_f1 = f1_score(y_train.cpu().numpy(), (train_preds.cpu().numpy() > threshold))\n",
    "        val_f1 = f1_score(y_val.cpu().numpy(), (val_preds.cpu().numpy() > threshold))\n",
    "\n",
    "        logger.write([epoch, optimizer.param_groups[0]['lr'], train_loss, val_loss, train_f1, val_f1])\n",
    "\n",
    "        print(f' {timer.fsince(epoch)} | End of Epoch {epoch} | Train Loss: {train_loss} | Val Loss: {val_loss} | Train F1: {round(train_f1, 4)} | Val F1: {round(val_f1, 4)}')\n",
    "\n",
    "        scheduler.step(val_loss)\n",
    "\n",
    "        if val_loss < best_val_loss:\n",
    "            best_epoch = epoch\n",
    "\n",
    "            best_train_loss = train_loss\n",
    "            best_val_loss = val_loss\n",
    "\n",
    "            best_train_f1 = train_f1\n",
    "            best_val_f1 = val_f1\n",
    "\n",
    "            save_checkpoint({\n",
    "                'model': model.state_dict(),\n",
    "                'optimizer': optimizer.state_dict(),\n",
    "                'epoch': epoch,\n",
    "                'train_iteration': train_iteration,\n",
    "                'val_iteration': val_iteration\n",
    "            })\n",
    "\n",
    "        elif epoch - best_epoch > early_stopping:\n",
    "            print(f' Val loss has not decreased for {early_stopping} epochs, stopping training')\n",
    "            break\n",
    "#     model = BaselineLSTM(pretrained_embedding)\n",
    "#     model = initialize(model)\n",
    "#     model.cuda()\n",
    "    preds = []\n",
    "\n",
    "#     model.eval()\n",
    "# #     with torch.no_grad():\n",
    "    for i, batch in enumerate(test_dataloader):\n",
    "        (question, length) = batch.question_text\n",
    "        k = pd.DataFrame(length.cpu().numpy(),columns=[\"length\"]).reset_index()\n",
    "        k = k.sort_values(by=\"length\",ascending=0)\n",
    "        out = model(question[k['index'].values], length[k['index'].values])\n",
    "        out = torch.sigmoid(out)\n",
    "        pred = out.detach().cpu().numpy()\n",
    "        mid_pred = np.zeros((pred.shape[0]))\n",
    "        for pred_of,mid_of in enumerate(k['index'].values):\n",
    "            mid_pred[mid_of] = pred[pred_of]\n",
    "        preds.extend(mid_pred)\n",
    "    all_train_preds[val_idx] = val_preds.cpu().numpy()[:,0]\n",
    "    real_valid_preds[val_idx] = y_val.cpu().numpy()[:,0]\n",
    "    test_preds_local[:, num_fold] = preds\n",
    "    preds = np.array(preds) * weight[num_fold] / sum(weight)\n",
    "    test_preds += preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "_uuid": "265663699f720cc3c80ce88d3f2df5f37d6fc823",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.937299</td>\n",
       "      <td>0.939289</td>\n",
       "      <td>0.932785</td>\n",
       "      <td>0.935587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.937299</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.941644</td>\n",
       "      <td>0.935862</td>\n",
       "      <td>0.940410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.939289</td>\n",
       "      <td>0.941644</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.940120</td>\n",
       "      <td>0.942462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.932785</td>\n",
       "      <td>0.935862</td>\n",
       "      <td>0.940120</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.939510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.935587</td>\n",
       "      <td>0.940410</td>\n",
       "      <td>0.942462</td>\n",
       "      <td>0.939510</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4\n",
       "0  1.000000  0.937299  0.939289  0.932785  0.935587\n",
       "1  0.937299  1.000000  0.941644  0.935862  0.940410\n",
       "2  0.939289  0.941644  1.000000  0.940120  0.942462\n",
       "3  0.932785  0.935862  0.940120  1.000000  0.939510\n",
       "4  0.935587  0.940410  0.942462  0.939510  1.000000"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(test_preds_local).corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "_uuid": "b037b4b8cc1b76d6bf52f709a7e069b2e03cb539",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def bestThresshold(y_train,train_preds):\n",
    "    tmp = [0,0,0] # idx, cur, max\n",
    "    delta = 0\n",
    "    for tmp[0] in np.arange(0.1, 0.901, 0.01):\n",
    "        tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])\n",
    "        if tmp[1] > tmp[2]:\n",
    "            delta = tmp[0]\n",
    "            tmp[2] = tmp[1]\n",
    "    print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))\n",
    "    return delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "_uuid": "cfc32803165188b4ba4ad47e0f40c59c95af59db"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "_uuid": "6d93dfad029210f27c4a80af177d4a490db2445d",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best threshold is 0.3700 with F1 score: 0.6835\n"
     ]
    }
   ],
   "source": [
    "thres = bestThresshold(real_valid_preds, sigmoid(all_train_preds))\n",
    "# y_train_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "_uuid": "ab3b71a5ac16ee9e5c54b4e266d1de46ee0a0c6d"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "_uuid": "3f5c296d79520402657f33f8c237dd74df0423ac"
   },
   "outputs": [],
   "source": [
    "submission = test[['qid']].copy()\n",
    "submission['prediction'] = (test_preds > thres).astype(int)\n",
    "submission.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "_uuid": "49231a2c9b66beeabfed7d3cb448019d1566ec00"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00014894849d00ba98a9</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000156468431f09b3cae</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000227734433360e1aae</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0005e06fbe3045bd2a92</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00068a0f7f41f50fc399</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    qid  prediction\n",
       "0  00014894849d00ba98a9           0\n",
       "1  000156468431f09b3cae           0\n",
       "2  000227734433360e1aae           0\n",
       "3  0005e06fbe3045bd2a92           0\n",
       "4  00068a0f7f41f50fc399           0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}