base_xgb_nn_cv5_PL.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Importing the libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import xgboost as xgb\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "\n",
    "train_df = pd.read_csv(\"./input/train.csv\")\n",
    "test_df = pd.read_csv(\"./input/test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing train...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 26/26 [00:26<00:00,  1.02s/it]\n",
      "100%|██████████| 26/26 [01:03<00:00,  2.45s/it]\n",
      "100%|██████████| 26/26 [07:41<00:00, 17.73s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing test...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 26/26 [00:11<00:00,  2.24it/s]\n",
      "100%|██████████| 26/26 [00:27<00:00,  1.07s/it]\n",
      "100%|██████████| 26/26 [03:38<00:00,  8.42s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19579, 26685) (8392, 26685)\n"
     ]
    }
   ],
   "source": [
    "# Clean text\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()\n",
    "punctuation = ['.', '..', '...', ',', ':', ';', '-', '*', '\"', '!', '?']\n",
    "alphabet = 'abcdefghijklmnopqrstuvwxyz'\n",
    "def clean_text(x):\n",
    "    x.lower()\n",
    "    for p in punctuation:\n",
    "        x.replace(p, '')\n",
    "    return x\n",
    "\n",
    "\n",
    "\n",
    "def extract_features(in_df, train_flag=False):\n",
    "    df = in_df.copy()\n",
    "    df['text_cleaned'] = df['text'].apply(lambda x: clean_text(x))\n",
    "    df['n_.'] = df['text'].str.count('\\.')\n",
    "    df['n_...'] = df['text'].str.count('\\...')\n",
    "    df['n_,'] = df['text'].str.count('\\,')\n",
    "    df['n_:'] = df['text'].str.count('\\:')\n",
    "    df['n_;'] = df['text'].str.count('\\;')\n",
    "    df['n_-'] = df['text'].str.count('\\-')\n",
    "    df['n_?'] = df['text'].str.count('\\?')\n",
    "    df['n_!'] = df['text'].str.count('\\!')\n",
    "    df['n_\\''] = df['text'].str.count('\\'')\n",
    "    df['n_\"'] = df['text'].str.count('\\\"')\n",
    "\n",
    "    # First words in a sentence\n",
    "    df['n_The '] = df['text'].str.count('The ')\n",
    "    df['n_I '] = df['text'].str.count('I ')\n",
    "    df['n_It '] = df['text'].str.count('It ')\n",
    "    df['n_He '] = df['text'].str.count('He ')\n",
    "    df['n_Me '] = df['text'].str.count('Me ')\n",
    "    df['n_She '] = df['text'].str.count('She ')\n",
    "    df['n_We '] = df['text'].str.count('We ')\n",
    "    df['n_They '] = df['text'].str.count('They ')\n",
    "    df['n_You '] = df['text'].str.count('You ')\n",
    "    df['n_the'] = df['text_cleaned'].str.count('the ')\n",
    "    df['n_ a '] = df['text_cleaned'].str.count(' a ')\n",
    "    df['n_appear'] = df['text_cleaned'].str.count('appear')\n",
    "    df['n_little'] = df['text_cleaned'].str.count('little')\n",
    "    df['n_was '] = df['text_cleaned'].str.count('was ')\n",
    "    df['n_one '] = df['text_cleaned'].str.count('one ')\n",
    "    df['n_two '] = df['text_cleaned'].str.count('two ')\n",
    "    df['n_three '] = df['text_cleaned'].str.count('three ')\n",
    "    df['n_ten '] = df['text_cleaned'].str.count('ten ')\n",
    "    df['n_is '] = df['text_cleaned'].str.count('is ')\n",
    "    df['n_are '] = df['text_cleaned'].str.count('are ')\n",
    "    df['n_ed'] = df['text_cleaned'].str.count('ed ')\n",
    "    df['n_however'] = df['text_cleaned'].str.count('however')\n",
    "    df['n_ to '] = df['text_cleaned'].str.count(' to ')\n",
    "    df['n_into'] = df['text_cleaned'].str.count('into')\n",
    "    df['n_about '] = df['text_cleaned'].str.count('about ')\n",
    "    df['n_th'] = df['text_cleaned'].str.count('th')\n",
    "    df['n_er'] = df['text_cleaned'].str.count('er')\n",
    "    df['n_ex'] = df['text_cleaned'].str.count('ex')\n",
    "    df['n_an '] = df['text_cleaned'].str.count('an ')\n",
    "    df['n_ground'] = df['text_cleaned'].str.count('ground')\n",
    "    df['n_any'] = df['text_cleaned'].str.count('any')\n",
    "    df['n_silence'] = df['text_cleaned'].str.count('silence')\n",
    "    df['n_wall'] = df['text_cleaned'].str.count('wall')\n",
    "    \n",
    "    # Find numbers of different combinations\n",
    "    for c in tqdm(alphabet.upper()):\n",
    "        df['n_' + c] = df['text'].str.count(c)\n",
    "        df['n_' + c + '.'] = df['text'].str.count(c + '\\.')\n",
    "        df['n_' + c + ','] = df['text'].str.count(c + '\\,')\n",
    "\n",
    "        for c2 in alphabet:\n",
    "            df['n_' + c + c2] = df['text'].str.count(c + c2)\n",
    "            df['n_' + c + c2 + '.'] = df['text'].str.count(c + c2 + '\\.')\n",
    "            df['n_' + c + c2 + ','] = df['text'].str.count(c + c2 + '\\,')\n",
    "\n",
    "    for c in tqdm(alphabet):\n",
    "        df['n_' + c + '.'] = df['text'].str.count(c + '\\.')\n",
    "        df['n_' + c + ','] = df['text'].str.count(c + '\\,')\n",
    "        df['n_' + c + '?'] = df['text'].str.count(c + '\\?')\n",
    "        df['n_' + c + ';'] = df['text'].str.count(c + '\\;')\n",
    "        df['n_' + c + ':'] = df['text'].str.count(c + '\\:')\n",
    "\n",
    "        for c2 in alphabet:\n",
    "            df['n_' + c + c2 + '.'] = df['text'].str.count(c + c2 + '\\.')\n",
    "            df['n_' + c + c2 + ','] = df['text'].str.count(c + c2 + '\\,')\n",
    "            df['n_' + c + c2 + '?'] = df['text'].str.count(c + c2 + '\\?')\n",
    "            df['n_' + c + c2 + ';'] = df['text'].str.count(c + c2 + '\\;')\n",
    "            df['n_' + c + c2 + ':'] = df['text'].str.count(c + c2 + '\\:')\n",
    "            df['n_' + c + ', ' + c2] = df['text'].str.count(c + '\\, ' + c2)\n",
    "\n",
    "    # And now starting processing of cleaned text\n",
    "    for c in tqdm(alphabet):\n",
    "        df['n_' + c] = df['text_cleaned'].str.count(c)\n",
    "        df['n_' + c + ' '] = df['text_cleaned'].str.count(c + ' ')\n",
    "        df['n_' + ' ' + c] = df['text_cleaned'].str.count(' ' + c)\n",
    "\n",
    "        for c2 in alphabet:\n",
    "            df['n_' + c + c2] = df['text_cleaned'].str.count(c + c2)\n",
    "            df['n_' + c + c2 + ' '] = df['text_cleaned'].str.count(c + c2 + ' ')\n",
    "            df['n_' + ' ' + c + c2] = df['text_cleaned'].str.count(' ' + c + c2)\n",
    "            df['n_' + c + ' ' + c2] = df['text_cleaned'].str.count(c + ' ' + c2)\n",
    "\n",
    "            for c3 in alphabet:\n",
    "                df['n_' + c + c2 + c3] = df['text_cleaned'].str.count(c + c2 + c3)\n",
    "                \n",
    "    if train_flag:\n",
    "        df.drop(['text_cleaned','text','author','id'], axis=1, inplace=True)\n",
    "    else:\n",
    "        df.drop(['text_cleaned','text','id'], axis=1, inplace=True)\n",
    "    return df.values\n",
    "    \n",
    "print('Processing train...')\n",
    "train_hand_features = extract_features(train_df,train_flag=True)\n",
    "print('Processing test...')\n",
    "test_hand_features = extract_features(test_df)\n",
    "print(train_hand_features.shape,test_hand_features.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# replace\n",
    "# train_df['text'] = train_df['text'].str.replace('[^a-zA-Z0-9]', ' ')\n",
    "# test_df['text'] =test_df['text'].str.replace('[^a-zA-Z0-9]', ' ')\n",
    "\n",
    "## Number of words in the text ##\n",
    "train_df[\"num_words\"] = train_df[\"text\"].apply(lambda x: len(str(x).split()))\n",
    "test_df[\"num_words\"] = test_df[\"text\"].apply(lambda x: len(str(x).split()))\n",
    "\n",
    "## Number of unique words in the text ##\n",
    "train_df[\"num_unique_words\"] = train_df[\"text\"].apply(lambda x: len(set(str(x).split())))\n",
    "test_df[\"num_unique_words\"] = test_df[\"text\"].apply(lambda x: len(set(str(x).split())))\n",
    "\n",
    "## Number of characters in the text ##\n",
    "train_df[\"num_chars\"] = train_df[\"text\"].apply(lambda x: len(str(x)))\n",
    "test_df[\"num_chars\"] = test_df[\"text\"].apply(lambda x: len(str(x)))\n",
    "\n",
    "## Number of stopwords in the text ##\n",
    "eng_stopwords = [\n",
    "    \"a\", \"about\", \"above\", \"across\", \"after\", \"afterwards\", \"again\", \"against\",\n",
    "    \"all\", \"almost\", \"alone\", \"along\", \"already\", \"also\", \"although\", \"always\",\n",
    "    \"am\", \"among\", \"amongst\", \"amoungst\", \"amount\", \"an\", \"and\", \"another\",\n",
    "    \"any\", \"anyhow\", \"anyone\", \"anything\", \"anyway\", \"anywhere\", \"are\",\n",
    "    \"around\", \"as\", \"at\", \"back\", \"be\", \"became\", \"because\", \"become\",\n",
    "    \"becomes\", \"becoming\", \"been\", \"before\", \"beforehand\", \"behind\", \"being\",\n",
    "    \"below\", \"beside\", \"besides\", \"between\", \"beyond\", \"bill\", \"both\",\n",
    "    \"bottom\", \"but\", \"by\", \"call\", \"can\", \"cannot\", \"cant\", \"co\", \"con\",\n",
    "    \"could\", \"couldnt\", \"cry\", \"de\", \"describe\", \"detail\", \"do\", \"done\",\n",
    "    \"down\", \"due\", \"during\", \"each\", \"eg\", \"eight\", \"either\", \"eleven\", \"else\",\n",
    "    \"elsewhere\", \"empty\", \"enough\", \"etc\", \"even\", \"ever\", \"every\", \"everyone\",\n",
    "    \"everything\", \"everywhere\", \"except\", \"few\", \"fifteen\", \"fifty\", \"fill\",\n",
    "    \"find\", \"fire\", \"first\", \"five\", \"for\", \"former\", \"formerly\", \"forty\",\n",
    "    \"found\", \"four\", \"from\", \"front\", \"full\", \"further\", \"get\", \"give\", \"go\",\n",
    "    \"had\", \"has\", \"hasnt\", \"have\", \"he\", \"hence\", \"her\", \"here\", \"hereafter\",\n",
    "    \"hereby\", \"herein\", \"hereupon\", \"hers\", \"herself\", \"him\", \"himself\", \"his\",\n",
    "    \"how\", \"however\", \"hundred\", \"i\", \"ie\", \"if\", \"in\", \"inc\", \"indeed\",\n",
    "    \"interest\", \"into\", \"is\", \"it\", \"its\", \"itself\", \"keep\", \"last\", \"latter\",\n",
    "    \"latterly\", \"least\", \"less\", \"ltd\", \"made\", \"many\", \"may\", \"me\",\n",
    "    \"meanwhile\", \"might\", \"mill\", \"mine\", \"more\", \"moreover\", \"most\", \"mostly\",\n",
    "    \"move\", \"much\", \"must\", \"my\", \"myself\", \"name\", \"namely\", \"neither\",\n",
    "    \"never\", \"nevertheless\", \"next\", \"nine\", \"no\", \"nobody\", \"none\", \"noone\",\n",
    "    \"nor\", \"not\", \"nothing\", \"now\", \"nowhere\", \"of\", \"off\", \"often\", \"on\",\n",
    "    \"once\", \"one\", \"only\", \"onto\", \"or\", \"other\", \"others\", \"otherwise\", \"our\",\n",
    "    \"ours\", \"ourselves\", \"out\", \"over\", \"own\", \"part\", \"per\", \"perhaps\",\n",
    "    \"please\", \"put\", \"rather\", \"re\", \"same\", \"see\", \"seem\", \"seemed\",\n",
    "    \"seeming\", \"seems\", \"serious\", \"several\", \"she\", \"should\", \"show\", \"side\",\n",
    "    \"since\", \"sincere\", \"six\", \"sixty\", \"so\", \"some\", \"somehow\", \"someone\",\n",
    "    \"something\", \"sometime\", \"sometimes\", \"somewhere\", \"still\", \"such\",\n",
    "    \"system\", \"take\", \"ten\", \"than\", \"that\", \"the\", \"their\", \"them\",\n",
    "    \"themselves\", \"then\", \"thence\", \"there\", \"thereafter\", \"thereby\",\n",
    "    \"therefore\", \"therein\", \"thereupon\", \"these\", \"they\", \"thick\", \"thin\",\n",
    "    \"third\", \"this\", \"those\", \"though\", \"three\", \"through\", \"throughout\",\n",
    "    \"thru\", \"thus\", \"to\", \"together\", \"too\", \"top\", \"toward\", \"towards\",\n",
    "    \"twelve\", \"twenty\", \"two\", \"un\", \"under\", \"until\", \"up\", \"upon\", \"us\",\n",
    "    \"very\", \"via\", \"was\", \"we\", \"well\", \"were\", \"what\", \"whatever\", \"when\",\n",
    "    \"whence\", \"whenever\", \"where\", \"whereafter\", \"whereas\", \"whereby\",\n",
    "    \"wherein\", \"whereupon\", \"wherever\", \"whether\", \"which\", \"while\", \"whither\",\n",
    "    \"who\", \"whoever\", \"whole\", \"whom\", \"whose\", \"why\", \"will\", \"with\",\n",
    "    \"within\", \"without\", \"would\", \"yet\", \"you\", \"your\", \"yours\", \"yourself\",\n",
    "    \"yourselves\"]\n",
    "train_df[\"num_stopwords\"] = train_df[\"text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in eng_stopwords]))\n",
    "test_df[\"num_stopwords\"] = test_df[\"text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in eng_stopwords]))\n",
    "\n",
    "## Number of punctuations in the text ##\n",
    "import string\n",
    "train_df[\"num_punctuations\"] =train_df['text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )\n",
    "test_df[\"num_punctuations\"] =test_df['text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )\n",
    "\n",
    "## Number of title case words in the text ##\n",
    "train_df[\"num_words_upper\"] = train_df[\"text\"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))\n",
    "test_df[\"num_words_upper\"] = test_df[\"text\"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))\n",
    "\n",
    "## Number of title case words in the text ##\n",
    "train_df[\"num_words_title\"] = train_df[\"text\"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))\n",
    "test_df[\"num_words_title\"] = test_df[\"text\"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))\n",
    "\n",
    "## Average length of the words in the text ##\n",
    "train_df[\"mean_word_len\"] = train_df[\"text\"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))\n",
    "test_df[\"mean_word_len\"] = test_df[\"text\"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>author</th>\n",
       "      <th>num_words</th>\n",
       "      <th>num_unique_words</th>\n",
       "      <th>num_chars</th>\n",
       "      <th>num_stopwords</th>\n",
       "      <th>num_punctuations</th>\n",
       "      <th>num_words_upper</th>\n",
       "      <th>num_words_title</th>\n",
       "      <th>mean_word_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id26305</td>\n",
       "      <td>This process, however, afforded me no means of...</td>\n",
       "      <td>EAP</td>\n",
       "      <td>41</td>\n",
       "      <td>35</td>\n",
       "      <td>231</td>\n",
       "      <td>23</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4.658537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id17569</td>\n",
       "      <td>It never once occurred to me that the fumbling...</td>\n",
       "      <td>HPL</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>71</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id11008</td>\n",
       "      <td>In his left hand was a gold snuff box, from wh...</td>\n",
       "      <td>EAP</td>\n",
       "      <td>36</td>\n",
       "      <td>32</td>\n",
       "      <td>200</td>\n",
       "      <td>16</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4.583333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id27763</td>\n",
       "      <td>How lovely is spring As we looked from Windsor...</td>\n",
       "      <td>MWS</td>\n",
       "      <td>34</td>\n",
       "      <td>32</td>\n",
       "      <td>206</td>\n",
       "      <td>14</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5.088235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id12958</td>\n",
       "      <td>Finding nothing else, not even gold, the Super...</td>\n",
       "      <td>HPL</td>\n",
       "      <td>27</td>\n",
       "      <td>25</td>\n",
       "      <td>174</td>\n",
       "      <td>13</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>5.481481</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                               text author  \\\n",
       "0  id26305  This process, however, afforded me no means of...    EAP   \n",
       "1  id17569  It never once occurred to me that the fumbling...    HPL   \n",
       "2  id11008  In his left hand was a gold snuff box, from wh...    EAP   \n",
       "3  id27763  How lovely is spring As we looked from Windsor...    MWS   \n",
       "4  id12958  Finding nothing else, not even gold, the Super...    HPL   \n",
       "\n",
       "   num_words  num_unique_words  num_chars  num_stopwords  num_punctuations  \\\n",
       "0         41                35        231             23                 7   \n",
       "1         14                14         71             10                 1   \n",
       "2         36                32        200             16                 5   \n",
       "3         34                32        206             14                 4   \n",
       "4         27                25        174             13                 4   \n",
       "\n",
       "   num_words_upper  num_words_title  mean_word_len  \n",
       "0                2                3       4.658537  \n",
       "1                0                1       4.142857  \n",
       "2                0                1       4.583333  \n",
       "3                0                4       5.088235  \n",
       "4                0                2       5.481481  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19579, 32) (8392, 32)\n",
      "(19579, 32) (8392, 32)\n",
      "(19579, 32) (8392, 32)\n",
      "(19579, 32) (8392, 32)\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "def pos_tag_text(s):\n",
    "    sents = nltk.sent_tokenize(s)\n",
    "    res = []\n",
    "    for sent in sents:\n",
    "        words = nltk.word_tokenize(sent)\n",
    "        tag_res = [a[1] for a in nltk.pos_tag(words)]\n",
    "        res.append(' '.join(tag_res))\n",
    "    return '. '.join(res)\n",
    "\n",
    "def ne_text(s):\n",
    "    sents = nltk.sent_tokenize(s)\n",
    "    res = []\n",
    "    for sent in sents:\n",
    "        words = nltk.word_tokenize(sent)\n",
    "        tag_res = nltk.pos_tag(words)\n",
    "        ne_tree = nltk.ne_chunk(tag_res)\n",
    "        list_res = nltk.tree2conlltags(ne_tree)\n",
    "        ne_res = [a[2] for a in list_res]\n",
    "        res.append(' '.join(ne_res))\n",
    "    return '. '.join(res)\n",
    "\n",
    "train_df['tag_txt'] = train_df[\"text\"].apply(pos_tag_text)\n",
    "train_df['ne_txt'] = train_df[\"text\"].apply(pos_tag_text)\n",
    "test_df['tag_txt'] = test_df[\"text\"].apply(pos_tag_text)\n",
    "test_df['ne_txt'] = test_df[\"text\"].apply(pos_tag_text)\n",
    "\n",
    "# cnt on tag\n",
    "c_vec3 = CountVectorizer(lowercase=False)\n",
    "c_vec3.fit(train_df['tag_txt'].values.tolist() + test_df['tag_txt'].values.tolist())\n",
    "train_cvec3 = c_vec3.transform(train_df['tag_txt'].values.tolist()).toarray()\n",
    "test_cvec3 = c_vec3.transform(test_df['tag_txt'].values.tolist()).toarray()\n",
    "print(train_cvec3.shape,test_cvec3.shape)\n",
    "\n",
    "\n",
    "# cnt on ne\n",
    "c_vec4 = CountVectorizer(lowercase=False)\n",
    "c_vec4.fit(train_df['ne_txt'].values.tolist() + test_df['ne_txt'].values.tolist())\n",
    "train_cvec4 = c_vec4.transform(train_df['ne_txt'].values.tolist()).toarray()\n",
    "test_cvec4 = c_vec4.transform(test_df['ne_txt'].values.tolist()).toarray()\n",
    "print(train_cvec4.shape,test_cvec4.shape)\n",
    "\n",
    "# cnt on tag\n",
    "tf_vec5 = TfidfVectorizer(lowercase=False)\n",
    "tf_vec5.fit(train_df['tag_txt'].values.tolist() + test_df['tag_txt'].values.tolist())\n",
    "train_tf5 = tf_vec5.transform(train_df['tag_txt'].values.tolist()).toarray()\n",
    "test_tf5 = tf_vec5.transform(test_df['tag_txt'].values.tolist()).toarray()\n",
    "print(train_tf5.shape,test_tf5.shape)\n",
    "\n",
    "\n",
    "# cnt on ne\n",
    "tf_vec6 = TfidfVectorizer(lowercase=False)\n",
    "tf_vec6.fit(train_df['ne_txt'].values.tolist() + test_df['ne_txt'].values.tolist())\n",
    "train_tf6 = tf_vec6.transform(train_df['ne_txt'].values.tolist()).toarray()\n",
    "test_tf6 = tf_vec6.transform(test_df['ne_txt'].values.tolist()).toarray()\n",
    "print(train_tf6.shape,test_tf6.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19579, 885129) (8392, 885129)\n",
      "(19579, 40) (8392, 40)\n",
      "(19579, 1354551) (8392, 1354551)\n",
      "(19579, 40) (8392, 40)\n",
      "(19579, 885129) (8392, 885129)\n",
      "(19579, 1354551) (8392, 1354551)\n",
      "(19579, 15) (8392, 15)\n"
     ]
    }
   ],
   "source": [
    "## Prepare the data for modeling ###\n",
    "author_mapping_dict = {'EAP':0, 'HPL':1, 'MWS':2}\n",
    "train_y = train_df['author'].map(author_mapping_dict)\n",
    "train_id = train_df['id'].values\n",
    "test_id = test_df['id'].values\n",
    "\n",
    "## add tfidf and svd\n",
    "tfidf_vec = TfidfVectorizer(ngram_range=(1,3), max_df=0.8,lowercase=False, sublinear_tf=True)\n",
    "full_tfidf = tfidf_vec.fit_transform(train_df['text'].values.tolist() + test_df['text'].values.tolist())\n",
    "train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())\n",
    "test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist())\n",
    "\n",
    "print(train_tfidf.shape,test_tfidf.shape)\n",
    "\n",
    "# svd1\n",
    "n_comp = 30\n",
    "svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')\n",
    "svd_obj.fit(full_tfidf)\n",
    "train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))\n",
    "test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))\n",
    "print(train_svd.shape,test_svd.shape)\n",
    "\n",
    "## add tfidf char\n",
    "tfidf_vec2 = TfidfVectorizer(ngram_range=(3,7), analyzer='char',max_df=0.8, sublinear_tf=True)\n",
    "full_tfidf2 = tfidf_vec2.fit_transform(train_df['text'].values.tolist() + test_df['text'].values.tolist())\n",
    "train_tfidf2 = tfidf_vec2.transform(train_df['text'].values.tolist())\n",
    "test_tfidf2 = tfidf_vec2.transform(test_df['text'].values.tolist())\n",
    "print(train_tfidf2.shape,test_tfidf2.shape)\n",
    "\n",
    "## add svd2\n",
    "n_comp = 30\n",
    "svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')\n",
    "svd_obj.fit(full_tfidf2)\n",
    "train_svd2 = pd.DataFrame(svd_obj.transform(train_tfidf2))\n",
    "test_svd2 = pd.DataFrame(svd_obj.transform(test_tfidf2))\n",
    "print(train_svd2.shape,test_svd2.shape)\n",
    "\n",
    "\n",
    "## add cnt vec\n",
    "c_vec = CountVectorizer(ngram_range=(1,3),max_df=0.8, lowercase=False)\n",
    "c_vec.fit(train_df['text'].values.tolist() + test_df['text'].values.tolist())\n",
    "train_cvec = c_vec.transform(train_df['text'].values.tolist())\n",
    "test_cvec = c_vec.transform(test_df['text'].values.tolist())\n",
    "print(train_cvec.shape,test_cvec.shape)\n",
    "\n",
    "# add cnt char\n",
    "c_vec2 = CountVectorizer(ngram_range=(3,7), analyzer='char',max_df=0.8)\n",
    "c_vec2.fit(train_df['text'].values.tolist() + test_df['text'].values.tolist())\n",
    "train_cvec2 = c_vec2.transform(train_df['text'].values.tolist())\n",
    "test_cvec2 = c_vec2.transform(test_df['text'].values.tolist())\n",
    "print(train_cvec2.shape,test_cvec2.shape)\n",
    "\n",
    "# add naive feature\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "feat_cnt = 3\n",
    "train_Y = train_y\n",
    "\n",
    "def gen_nb_feats(rnd=1):\n",
    "    help_tfidf_train,help_tfidf_test = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "    help_tfidf_train2,help_tfidf_test2 = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "    help_cnt1_train,help_cnt1_test = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "    help_cnt2_train,help_cnt2_test = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "    hand_train, hand_test = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "\n",
    "    kf = KFold(n_splits=feat_cnt, shuffle=True, random_state=23*rnd)\n",
    "    for train_index, test_index in kf.split(train_tfidf):\n",
    "        # tfidf to nb\n",
    "        X_train, X_test = train_tfidf[train_index], train_tfidf[test_index]\n",
    "        y_train, y_test = train_Y[train_index], train_Y[test_index]\n",
    "        tmp_model = MultinomialNB(alpha=0.025,fit_prior=False)\n",
    "        tmp_model.fit(X_train,y_train)\n",
    "        tmp_train_feat = tmp_model.predict_proba(X_test)\n",
    "        tmp_test_feat = tmp_model.predict_proba(test_tfidf)\n",
    "        help_tfidf_train[test_index] = tmp_train_feat\n",
    "        help_tfidf_test += tmp_test_feat/feat_cnt\n",
    "\n",
    "        # tfidf to nb\n",
    "        X_train, X_test = train_tfidf2[train_index], train_tfidf2[test_index]\n",
    "        tmp_model = MultinomialNB(0.025,fit_prior=False)\n",
    "        tmp_model.fit(X_train,y_train)\n",
    "        tmp_train_feat = tmp_model.predict_proba(X_test)\n",
    "        tmp_test_feat = tmp_model.predict_proba(test_tfidf2)\n",
    "        help_tfidf_train2[test_index] = tmp_train_feat\n",
    "        help_tfidf_test2 += tmp_test_feat/feat_cnt\n",
    "\n",
    "        # count vec to nb\n",
    "        X_train, X_test = train_cvec[train_index], train_cvec[test_index]\n",
    "        tmp_model = MultinomialNB(0.025,fit_prior=False)\n",
    "        tmp_model.fit(X_train,y_train)\n",
    "        tmp_train_feat = tmp_model.predict_proba(X_test)\n",
    "        tmp_test_feat = tmp_model.predict_proba(test_cvec)\n",
    "        help_cnt1_train[test_index] = tmp_train_feat\n",
    "        help_cnt1_test += tmp_test_feat/feat_cnt\n",
    "\n",
    "        # count vec2 to nb \n",
    "        X_train, X_test = train_cvec2[train_index], train_cvec2[test_index]\n",
    "        tmp_model = MultinomialNB(0.025,fit_prior=False)\n",
    "        tmp_model.fit(X_train,y_train)\n",
    "        tmp_train_feat = tmp_model.predict_proba(X_test)\n",
    "        tmp_test_feat = tmp_model.predict_proba(test_cvec2)\n",
    "        help_cnt2_train[test_index] = tmp_train_feat\n",
    "        help_cnt2_test += tmp_test_feat/feat_cnt\n",
    "        \n",
    "        # hand feature to nb\n",
    "        X_train, X_test = train_hand_features[train_index], train_hand_features[test_index]\n",
    "        tmp_model = MultinomialNB(0.025,fit_prior=False)\n",
    "        tmp_model.fit(X_train,y_train)\n",
    "        tmp_train_feat = tmp_model.predict_proba(X_test)\n",
    "        tmp_test_feat = tmp_model.predict_proba(test_hand_features)\n",
    "        hand_train[test_index] = tmp_train_feat\n",
    "        hand_test += tmp_test_feat/feat_cnt\n",
    "    \n",
    "    help_train_feat = np.hstack([help_tfidf_train,help_tfidf_train2,help_cnt1_train,help_cnt2_train,hand_train])\n",
    "    help_test_feat = np.hstack([help_tfidf_test,help_tfidf_test2,help_cnt1_test,help_cnt2_test,hand_test])\n",
    "\n",
    "    return help_train_feat,help_test_feat\n",
    "    \n",
    "help_train_feat,help_test_feat = gen_nb_feats(1)\n",
    "print(help_train_feat.shape,help_test_feat.shape)\n",
    "help_train_feat2,help_test_feat2 = gen_nb_feats(2)\n",
    "help_train_feat3,help_test_feat3 = gen_nb_feats(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19579, 3) (19579, 3)\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "from sklearn.mixture import BayesianGaussianMixture, GaussianMixture\n",
    "\n",
    "tmp_train_svd = np.hstack([train_svd,train_svd2])\n",
    "tmp_test_svd = np.hstack([test_svd,test_svd2])\n",
    "\n",
    "# svd to generative models\n",
    "\n",
    "def gen_by_gauss_feats(rnd=1):\n",
    "    feat_cnt = 3\n",
    "    hand_train, hand_test = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "    kf = KFold(n_splits=feat_cnt, shuffle=True, random_state=23*rnd)\n",
    "    for train_index, test_index in kf.split(tmp_train_svd):\n",
    "        # hand feature \n",
    "        X_train, X_test = tmp_train_svd[train_index], tmp_train_svd[test_index]\n",
    "        y_train, y_test = train_Y[train_index], train_Y[test_index]\n",
    "        tmp_model = BayesianGaussianMixture()\n",
    "        tmp_model.fit(X_train,y_train)\n",
    "        tmp_train_feat = tmp_model.predict_proba(X_test)\n",
    "        tmp_test_feat = tmp_model.predict_proba(tmp_test_svd)\n",
    "        hand_train[test_index] = tmp_train_feat\n",
    "        hand_test += tmp_test_feat/feat_cnt\n",
    "    \n",
    "    return hand_train,hand_test\n",
    "\n",
    "help_train_feat4,help_test_feat4 = gen_by_gauss_feats(1)\n",
    "\n",
    "def gen_gauss_feats(rnd=1):\n",
    "    feat_cnt = 3\n",
    "    hand_train, hand_test = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "    kf = KFold(n_splits=feat_cnt, shuffle=True, random_state=23*rnd)\n",
    "    for train_index, test_index in kf.split(tmp_train_svd):\n",
    "        # hand feature \n",
    "        X_train, X_test = tmp_train_svd[train_index], tmp_train_svd[test_index]\n",
    "        y_train, y_test = train_Y[train_index], train_Y[test_index]\n",
    "        tmp_model = GaussianMixture()\n",
    "        tmp_model.fit(X_train,y_train)\n",
    "        tmp_train_feat = tmp_model.predict_proba(X_test)\n",
    "        tmp_test_feat = tmp_model.predict_proba(tmp_test_svd)\n",
    "        hand_train[test_index] = tmp_train_feat\n",
    "        hand_test += tmp_test_feat/feat_cnt\n",
    "    \n",
    "    return hand_train,hand_test\n",
    "\n",
    "help_train_feat5,help_test_feat5 = gen_gauss_feats(1)\n",
    "\n",
    "print(help_train_feat4.shape, help_train_feat5.shape)\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "import keras done\n"
     ]
    }
   ],
   "source": [
    "# add cnn feat\n",
    "from keras.layers import Embedding, CuDNNLSTM, Dense, Flatten, Dropout, LSTM\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.layers import Conv1D, GlobalMaxPooling1D, GlobalAveragePooling1D\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import log_loss\n",
    "import gc\n",
    "print('import keras done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def cnn done\n",
      "Train on 11746 samples, validate on 1306 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 1.04954, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 1.0835 - acc: 0.4190 - val_loss: 1.0495 - val_acc: 0.4916\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 1.04954 to 0.86437, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.9549 - acc: 0.5462 - val_loss: 0.8644 - val_acc: 0.5926\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.86437 to 0.75724, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.7614 - acc: 0.6515 - val_loss: 0.7572 - val_acc: 0.6501\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss improved from 0.75724 to 0.70945, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.6247 - acc: 0.7292 - val_loss: 0.7094 - val_acc: 0.6761\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss did not improve\n",
      " - 1s - loss: 0.5231 - acc: 0.7902 - val_loss: 0.7104 - val_acc: 0.6646\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 1s - loss: 0.4478 - acc: 0.8288 - val_loss: 0.7266 - val_acc: 0.6815\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 1s - loss: 0.3811 - acc: 0.8611 - val_loss: 0.7653 - val_acc: 0.6853\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 2s - loss: 0.3283 - acc: 0.8880 - val_loss: 0.8116 - val_acc: 0.6792\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 2s - loss: 0.2908 - acc: 0.9069 - val_loss: 0.8664 - val_acc: 0.6723\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 2s - loss: 0.2579 - acc: 0.9153 - val_loss: 0.9328 - val_acc: 0.6639\n",
      "------------------\n",
      "Train on 11747 samples, validate on 1306 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 1.06534, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 1.0845 - acc: 0.4059 - val_loss: 1.0653 - val_acc: 0.4319\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 1.06534 to 0.89575, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.9888 - acc: 0.5226 - val_loss: 0.8957 - val_acc: 0.6187\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.89575 to 0.64849, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.7375 - acc: 0.7164 - val_loss: 0.6485 - val_acc: 0.7626\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss improved from 0.64849 to 0.50453, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.4849 - acc: 0.8375 - val_loss: 0.5045 - val_acc: 0.8047\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss improved from 0.50453 to 0.45090, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.3418 - acc: 0.8863 - val_loss: 0.4509 - val_acc: 0.8224\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss improved from 0.45090 to 0.43087, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.2557 - acc: 0.9159 - val_loss: 0.4309 - val_acc: 0.8224\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss improved from 0.43087 to 0.42983, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.1988 - acc: 0.9371 - val_loss: 0.4298 - val_acc: 0.8208\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 2s - loss: 0.1549 - acc: 0.9523 - val_loss: 0.4409 - val_acc: 0.8231\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 2s - loss: 0.1258 - acc: 0.9622 - val_loss: 0.4559 - val_acc: 0.8247\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 2s - loss: 0.1047 - acc: 0.9694 - val_loss: 0.4760 - val_acc: 0.8262\n",
      "------------------\n",
      "Train on 11747 samples, validate on 1306 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 1.06207, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 1.0851 - acc: 0.4085 - val_loss: 1.0621 - val_acc: 0.4495\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 1.06207 to 0.85236, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.9761 - acc: 0.5269 - val_loss: 0.8524 - val_acc: 0.6271\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.85236 to 0.61110, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.7159 - acc: 0.7207 - val_loss: 0.6111 - val_acc: 0.7718\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss improved from 0.61110 to 0.47087, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.4741 - acc: 0.8361 - val_loss: 0.4709 - val_acc: 0.8308\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss improved from 0.47087 to 0.41857, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.3301 - acc: 0.8890 - val_loss: 0.4186 - val_acc: 0.8469\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss improved from 0.41857 to 0.40430, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.2453 - acc: 0.9224 - val_loss: 0.4043 - val_acc: 0.8560\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 2s - loss: 0.1889 - acc: 0.9404 - val_loss: 0.4049 - val_acc: 0.8499\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 2s - loss: 0.1494 - acc: 0.9560 - val_loss: 0.4195 - val_acc: 0.8499\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 2s - loss: 0.1213 - acc: 0.9642 - val_loss: 0.4303 - val_acc: 0.8423\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 2s - loss: 0.0992 - acc: 0.9732 - val_loss: 0.4542 - val_acc: 0.8369\n",
      "------------------\n",
      "Train on 11746 samples, validate on 1306 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 1.05593, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 1.0834 - acc: 0.4107 - val_loss: 1.0559 - val_acc: 0.4518\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 1.05593 to 0.85299, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.9707 - acc: 0.5431 - val_loss: 0.8530 - val_acc: 0.6325\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.85299 to 0.60499, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.6883 - acc: 0.7441 - val_loss: 0.6050 - val_acc: 0.7511\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss improved from 0.60499 to 0.48275, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.4471 - acc: 0.8537 - val_loss: 0.4828 - val_acc: 0.8009\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss improved from 0.48275 to 0.44246, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.3098 - acc: 0.8989 - val_loss: 0.4425 - val_acc: 0.8224\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss improved from 0.44246 to 0.43218, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.2316 - acc: 0.9259 - val_loss: 0.4322 - val_acc: 0.8239\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 2s - loss: 0.1790 - acc: 0.9432 - val_loss: 0.4367 - val_acc: 0.8254\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 2s - loss: 0.1414 - acc: 0.9562 - val_loss: 0.4493 - val_acc: 0.8262\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 2s - loss: 0.1128 - acc: 0.9670 - val_loss: 0.4747 - val_acc: 0.8254\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 2s - loss: 0.0926 - acc: 0.9737 - val_loss: 0.5030 - val_acc: 0.8254\n",
      "------------------\n",
      "Train on 11747 samples, validate on 1306 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 1.06966, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 1.0851 - acc: 0.4075 - val_loss: 1.0697 - val_acc: 0.4173\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 1.06966 to 0.86759, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.9802 - acc: 0.5407 - val_loss: 0.8676 - val_acc: 0.6570\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.86759 to 0.59060, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.6889 - acc: 0.7459 - val_loss: 0.5906 - val_acc: 0.7841\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss improved from 0.59060 to 0.46374, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.4349 - acc: 0.8553 - val_loss: 0.4637 - val_acc: 0.8124\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss improved from 0.46374 to 0.41893, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.3018 - acc: 0.8994 - val_loss: 0.4189 - val_acc: 0.8407\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss improved from 0.41893 to 0.40728, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.2235 - acc: 0.9300 - val_loss: 0.4073 - val_acc: 0.8453\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 2s - loss: 0.1738 - acc: 0.9468 - val_loss: 0.4126 - val_acc: 0.8461\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 2s - loss: 0.1356 - acc: 0.9602 - val_loss: 0.4287 - val_acc: 0.8484\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 2s - loss: 0.1106 - acc: 0.9683 - val_loss: 0.4481 - val_acc: 0.8461\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 2s - loss: 0.0909 - acc: 0.9752 - val_loss: 0.4714 - val_acc: 0.8469\n",
      "------------------\n",
      "Train on 11747 samples, validate on 1306 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 1.06997, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 1.0867 - acc: 0.4033 - val_loss: 1.0700 - val_acc: 0.4257\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 1.06997 to 0.85703, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.9753 - acc: 0.5424 - val_loss: 0.8570 - val_acc: 0.6631\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.85703 to 0.60855, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.7019 - acc: 0.7364 - val_loss: 0.6085 - val_acc: 0.7971\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss improved from 0.60855 to 0.48435, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.4504 - acc: 0.8483 - val_loss: 0.4844 - val_acc: 0.8170\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss improved from 0.48435 to 0.44386, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.3193 - acc: 0.8936 - val_loss: 0.4439 - val_acc: 0.8323\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss improved from 0.44386 to 0.42527, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.2383 - acc: 0.9227 - val_loss: 0.4253 - val_acc: 0.8377\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 2s - loss: 0.1833 - acc: 0.9418 - val_loss: 0.4294 - val_acc: 0.8361\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 2s - loss: 0.1474 - acc: 0.9538 - val_loss: 0.4425 - val_acc: 0.8346\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 2s - loss: 0.1196 - acc: 0.9649 - val_loss: 0.4688 - val_acc: 0.8300\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 2s - loss: 0.0971 - acc: 0.9734 - val_loss: 0.4844 - val_acc: 0.8346\n",
      "------------------\n",
      "Train on 11746 samples, validate on 1306 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 1.06831, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 1.0858 - acc: 0.4051 - val_loss: 1.0683 - val_acc: 0.4265\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 1.06831 to 0.88179, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.9835 - acc: 0.5313 - val_loss: 0.8818 - val_acc: 0.5904\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.88179 to 0.75140, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.7892 - acc: 0.6319 - val_loss: 0.7514 - val_acc: 0.6685\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss improved from 0.75140 to 0.66167, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.6297 - acc: 0.7394 - val_loss: 0.6617 - val_acc: 0.7282\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss improved from 0.66167 to 0.57592, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.4816 - acc: 0.8337 - val_loss: 0.5759 - val_acc: 0.7711\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss improved from 0.57592 to 0.52425, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.3544 - acc: 0.8850 - val_loss: 0.5243 - val_acc: 0.7948\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss improved from 0.52425 to 0.49288, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.2648 - acc: 0.9181 - val_loss: 0.4929 - val_acc: 0.8078\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 2s - loss: 0.2061 - acc: 0.9377 - val_loss: 0.4948 - val_acc: 0.8147\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 2s - loss: 0.1630 - acc: 0.9515 - val_loss: 0.5045 - val_acc: 0.8155\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 2s - loss: 0.1345 - acc: 0.9619 - val_loss: 0.5124 - val_acc: 0.8224\n",
      "------------------\n",
      "Train on 11747 samples, validate on 1306 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 1.06648, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 1.0848 - acc: 0.4057 - val_loss: 1.0665 - val_acc: 0.4364\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 1.06648 to 0.90908, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.9877 - acc: 0.5166 - val_loss: 0.9091 - val_acc: 0.5873\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.90908 to 0.67364, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.7638 - acc: 0.6984 - val_loss: 0.6736 - val_acc: 0.7519\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss improved from 0.67364 to 0.50512, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.5037 - acc: 0.8283 - val_loss: 0.5051 - val_acc: 0.8025\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss improved from 0.50512 to 0.43994, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.3416 - acc: 0.8857 - val_loss: 0.4399 - val_acc: 0.8308\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss improved from 0.43994 to 0.42142, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.2519 - acc: 0.9173 - val_loss: 0.4214 - val_acc: 0.8361\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 2s - loss: 0.1874 - acc: 0.9427 - val_loss: 0.4268 - val_acc: 0.8354\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 2s - loss: 0.1528 - acc: 0.9547 - val_loss: 0.4373 - val_acc: 0.8377\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 2s - loss: 0.1188 - acc: 0.9649 - val_loss: 0.4564 - val_acc: 0.8384\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 2s - loss: 0.0989 - acc: 0.9732 - val_loss: 0.4760 - val_acc: 0.8361\n",
      "------------------\n",
      "Train on 11747 samples, validate on 1306 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 1.06680, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 1.0853 - acc: 0.4061 - val_loss: 1.0668 - val_acc: 0.4219\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 1.06680 to 0.82895, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.9631 - acc: 0.5578 - val_loss: 0.8289 - val_acc: 0.6884\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.82895 to 0.57820, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.6547 - acc: 0.7643 - val_loss: 0.5782 - val_acc: 0.7848\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss improved from 0.57820 to 0.46930, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.4266 - acc: 0.8522 - val_loss: 0.4693 - val_acc: 0.8155\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss improved from 0.46930 to 0.42988, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.3053 - acc: 0.8959 - val_loss: 0.4299 - val_acc: 0.8315\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss improved from 0.42988 to 0.42057, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.2329 - acc: 0.9251 - val_loss: 0.4206 - val_acc: 0.8430\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 2s - loss: 0.1808 - acc: 0.9413 - val_loss: 0.4278 - val_acc: 0.8377\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 2s - loss: 0.1431 - acc: 0.9571 - val_loss: 0.4426 - val_acc: 0.8453\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 2s - loss: 0.1182 - acc: 0.9641 - val_loss: 0.4629 - val_acc: 0.8392\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 2s - loss: 0.0983 - acc: 0.9703 - val_loss: 0.4920 - val_acc: 0.8285\n",
      "------------------\n"
     ]
    }
   ],
   "source": [
    "def get_nn_feats(rnd=1):\n",
    "    # return train pred prob and test pred prob \n",
    "    train_pred, test_pred = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "    best_val_train_pred, best_val_test_pred = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "    FEAT_CNT = 3\n",
    "    NUM_WORDS = 30000\n",
    "    N = 10\n",
    "    MAX_LEN = 150\n",
    "    NUM_CLASSES = 3\n",
    "    MODEL_P = '/tmp/nn_model.h5'\n",
    "    \n",
    "    tmp_X = train_df['text']\n",
    "    tmp_Y = train_df['author']\n",
    "    tmp_X_test = test_df['text']\n",
    "    \n",
    "    tokenizer = Tokenizer(num_words=NUM_WORDS)\n",
    "    tokenizer.fit_on_texts(tmp_X)\n",
    "\n",
    "    ttrain_x = tokenizer.texts_to_sequences(tmp_X)\n",
    "    ttrain_x = pad_sequences(ttrain_x, maxlen=MAX_LEN)\n",
    "    \n",
    "    ttest_x = tokenizer.texts_to_sequences(tmp_X_test)\n",
    "    ttest_x = pad_sequences(ttest_x, maxlen=MAX_LEN)\n",
    "\n",
    "    lb = preprocessing.LabelBinarizer()\n",
    "    lb.fit(tmp_Y)\n",
    "\n",
    "    ttrain_y = lb.transform(tmp_Y)\n",
    "    kf = KFold(n_splits=FEAT_CNT, shuffle=True, random_state=233*rnd)\n",
    "    for train_index, test_index in kf.split(train_tfidf):\n",
    "        model = Sequential()\n",
    "        model.add(Embedding(NUM_WORDS, N, input_length=MAX_LEN))\n",
    "        model.add(GlobalAveragePooling1D())\n",
    "        model.add(Dense(30, activation='relu'))\n",
    "        model.add(Dropout(0.1))\n",
    "        model.add(Dense(NUM_CLASSES, activation='softmax'))\n",
    "\n",
    "        model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "        #model.summary()\n",
    "\n",
    "        model_chk = ModelCheckpoint(filepath=MODEL_P, monitor='val_loss', save_best_only=True, verbose=1)\n",
    "        np.random.seed(42) # for model train\n",
    "        model.fit(ttrain_x[train_index], ttrain_y[train_index], \n",
    "                  validation_split=0.1,\n",
    "                  batch_size=64, epochs=10, \n",
    "                  verbose=2,\n",
    "                  callbacks=[model_chk],\n",
    "                  shuffle=False\n",
    "                 )\n",
    " \n",
    "        # save feat\n",
    "        train_pred[test_index] = model.predict(ttrain_x[test_index])\n",
    "        test_pred += model.predict(ttest_x)/feat_cnt\n",
    "        \n",
    "        # best val model\n",
    "        model = load_model(MODEL_P)\n",
    "        best_val_train_pred[test_index] = model.predict(ttrain_x[test_index])\n",
    "        best_val_test_pred += model.predict(ttest_x)/feat_cnt\n",
    "        \n",
    "        # release\n",
    "        del model\n",
    "        gc.collect()\n",
    "        print('------------------')\n",
    "        \n",
    "    return train_pred,test_pred,best_val_train_pred,best_val_test_pred\n",
    "\n",
    "print('def cnn done')\n",
    "\n",
    "cnn_train1,cnn_test1,cnn_train2,cnn_test2 = get_nn_feats(1)\n",
    "cnn_train3,cnn_test3,cnn_train4,cnn_test4 = get_nn_feats(2)\n",
    "cnn_train5,cnn_test5,cnn_train6,cnn_test6 = get_nn_feats(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def lstm done\n",
      "Train on 11746 samples, validate on 1306 samples\n",
      "Epoch 1/6\n",
      "Epoch 00001: val_loss improved from inf to 0.74357, saving model to /tmp/nn_model.h5\n",
      " - 26s - loss: 0.9879 - acc: 0.4935 - val_loss: 0.7436 - val_acc: 0.6922\n",
      "Epoch 2/6\n",
      "Epoch 00002: val_loss improved from 0.74357 to 0.47593, saving model to /tmp/nn_model.h5\n",
      " - 25s - loss: 0.4874 - acc: 0.8107 - val_loss: 0.4759 - val_acc: 0.8032\n",
      "Epoch 3/6\n",
      "Epoch 00003: val_loss did not improve\n",
      " - 27s - loss: 0.2572 - acc: 0.9052 - val_loss: 0.4870 - val_acc: 0.8254\n",
      "Epoch 4/6\n",
      "Epoch 00004: val_loss did not improve\n",
      " - 25s - loss: 0.1612 - acc: 0.9436 - val_loss: 0.5467 - val_acc: 0.8239\n",
      "Epoch 5/6\n",
      "Epoch 00005: val_loss did not improve\n",
      " - 26s - loss: 0.1082 - acc: 0.9637 - val_loss: 0.6122 - val_acc: 0.8193\n",
      "Epoch 6/6\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 27s - loss: 0.0717 - acc: 0.9757 - val_loss: 0.7405 - val_acc: 0.8124\n",
      "------------------\n",
      "Train on 11747 samples, validate on 1306 samples\n",
      "Epoch 1/6\n",
      "Epoch 00001: val_loss improved from inf to 0.62116, saving model to /tmp/nn_model.h5\n",
      " - 29s - loss: 0.9330 - acc: 0.5422 - val_loss: 0.6212 - val_acc: 0.7404\n",
      "Epoch 2/6\n",
      "Epoch 00002: val_loss improved from 0.62116 to 0.46862, saving model to /tmp/nn_model.h5\n",
      " - 27s - loss: 0.4422 - acc: 0.8296 - val_loss: 0.4686 - val_acc: 0.8239\n",
      "Epoch 3/6\n",
      "Epoch 00003: val_loss did not improve\n",
      " - 28s - loss: 0.2424 - acc: 0.9139 - val_loss: 0.4736 - val_acc: 0.8178\n",
      "Epoch 4/6\n",
      "Epoch 00004: val_loss did not improve\n",
      " - 27s - loss: 0.1516 - acc: 0.9467 - val_loss: 0.5470 - val_acc: 0.8132\n",
      "Epoch 5/6\n",
      "Epoch 00005: val_loss did not improve\n",
      " - 25s - loss: 0.1017 - acc: 0.9643 - val_loss: 0.6338 - val_acc: 0.8109\n",
      "Epoch 6/6\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 25s - loss: 0.0728 - acc: 0.9752 - val_loss: 0.7194 - val_acc: 0.8047\n",
      "------------------\n",
      "Train on 11747 samples, validate on 1306 samples\n",
      "Epoch 1/6\n",
      "Epoch 00001: val_loss improved from inf to 0.74293, saving model to /tmp/nn_model.h5\n",
      " - 27s - loss: 0.9603 - acc: 0.5163 - val_loss: 0.7429 - val_acc: 0.6677\n",
      "Epoch 2/6\n",
      "Epoch 00002: val_loss improved from 0.74293 to 0.46641, saving model to /tmp/nn_model.h5\n",
      " - 25s - loss: 0.5004 - acc: 0.8000 - val_loss: 0.4664 - val_acc: 0.8247\n",
      "Epoch 3/6\n",
      "Epoch 00003: val_loss improved from 0.46641 to 0.44858, saving model to /tmp/nn_model.h5\n",
      " - 25s - loss: 0.2627 - acc: 0.9048 - val_loss: 0.4486 - val_acc: 0.8315\n",
      "Epoch 4/6\n",
      "Epoch 00004: val_loss did not improve\n",
      " - 25s - loss: 0.1554 - acc: 0.9468 - val_loss: 0.5031 - val_acc: 0.8270\n",
      "Epoch 5/6\n",
      "Epoch 00005: val_loss did not improve\n",
      " - 25s - loss: 0.1106 - acc: 0.9620 - val_loss: 0.5865 - val_acc: 0.8239\n",
      "Epoch 6/6\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 25s - loss: 0.0742 - acc: 0.9747 - val_loss: 0.7032 - val_acc: 0.8139\n",
      "------------------\n"
     ]
    }
   ],
   "source": [
    "# add lstm feat\n",
    "def get_lstm_feats(rnd=1):\n",
    "    # return train pred prob and test pred prob \n",
    "    train_pred, test_pred = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "    best_val_train_pred, best_val_test_pred = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "    FEAT_CNT = 3\n",
    "    NUM_WORDS = 16000\n",
    "    N = 12\n",
    "    MAX_LEN = 300\n",
    "    NUM_CLASSES = 3\n",
    "    MODEL_P = '/tmp/nn_model.h5'\n",
    "    \n",
    "    tmp_X = train_df['text']\n",
    "    tmp_Y = train_df['author']\n",
    "    tmp_X_test = test_df['text']\n",
    "    \n",
    "    tokenizer = Tokenizer(num_words=NUM_WORDS)\n",
    "    tokenizer.fit_on_texts(tmp_X)\n",
    "\n",
    "    ttrain_x = tokenizer.texts_to_sequences(tmp_X)\n",
    "    ttrain_x = pad_sequences(ttrain_x, maxlen=MAX_LEN)\n",
    "    \n",
    "    ttest_x = tokenizer.texts_to_sequences(tmp_X_test)\n",
    "    ttest_x = pad_sequences(ttest_x, maxlen=MAX_LEN)\n",
    "\n",
    "    lb = preprocessing.LabelBinarizer()\n",
    "    lb.fit(tmp_Y)\n",
    "\n",
    "    ttrain_y = lb.transform(tmp_Y)\n",
    "    kf = KFold(n_splits=FEAT_CNT, shuffle=True, random_state=2333*rnd)\n",
    "    for train_index, test_index in kf.split(train_tfidf):\n",
    "        model = Sequential()\n",
    "        model.add(Embedding(NUM_WORDS, N, input_length=MAX_LEN))\n",
    "        model.add(LSTM(N, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))\n",
    "        model.add(Flatten())\n",
    "        model.add(Dense(128, activation='relu'))\n",
    "        model.add(Dropout(0.2))\n",
    "        model.add(Dense(NUM_CLASSES, activation='softmax'))\n",
    "        model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "        #model.summary()\n",
    "\n",
    "        model_chk = ModelCheckpoint(filepath=MODEL_P, monitor='val_loss', save_best_only=True, verbose=1)\n",
    "        np.random.seed(42) # for model train\n",
    "        model.fit(ttrain_x[train_index], ttrain_y[train_index], \n",
    "                  validation_split=0.1,\n",
    "                  batch_size=64, epochs=6, \n",
    "                  verbose=2,\n",
    "                  callbacks=[model_chk],\n",
    "                  shuffle=False\n",
    "                 )\n",
    "        # save feat\n",
    "        train_pred[test_index] = model.predict(ttrain_x[test_index])\n",
    "        test_pred += model.predict(ttest_x)/feat_cnt\n",
    "        \n",
    "        # best val model\n",
    "        model = load_model(MODEL_P)\n",
    "        best_val_train_pred[test_index] = model.predict(ttrain_x[test_index])\n",
    "        best_val_test_pred += model.predict(ttest_x)/feat_cnt\n",
    "        \n",
    "        del model\n",
    "        gc.collect()\n",
    "        print('------------------')\n",
    "        \n",
    "    return train_pred,test_pred,best_val_train_pred,best_val_test_pred\n",
    "\n",
    "print('def lstm done')\n",
    "lstm_train1,lstm_test1,lstm_train2,lstm_test2 = get_lstm_feats(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19579, 291) (8392, 291)\n"
     ]
    }
   ],
   "source": [
    "# combine feats\n",
    "cols_to_drop = ['id', 'text','tag_txt','ne_txt']\n",
    "train_X = train_df.drop(cols_to_drop+['author'], axis=1).values\n",
    "test_X = test_df.drop(cols_to_drop, axis=1).values\n",
    "train_X = np.hstack([train_X,train_svd,train_svd2,train_cvec3,train_cvec4,train_tf5,train_tf6])\n",
    "test_X = np.hstack([test_X,test_svd,test_svd2,test_cvec3,test_cvec4,test_tf5,test_tf6])\n",
    "\n",
    "f_train_X = np.hstack([train_X, help_train_feat,help_train_feat2,\n",
    "                       help_train_feat3,help_train_feat4,help_train_feat5,\n",
    "                       lstm_train1, lstm_train2, cnn_train1, cnn_train2,\n",
    "                       cnn_train3, cnn_train4,cnn_train5, cnn_train6])\n",
    "f_train_X = np.round(f_train_X,4)\n",
    "\n",
    "\n",
    "f_test_X = np.hstack([test_X, help_test_feat,help_test_feat2,help_test_feat3,\n",
    "                      help_test_feat4,help_test_feat5,\n",
    "                      lstm_test1, lstm_test2, cnn_test1, cnn_test2,\n",
    "                      cnn_test3, cnn_test4,cnn_test5, cnn_test6\n",
    "                     ])\n",
    "f_test_X = np.round(f_test_X,4)\n",
    "print(f_train_X.shape, f_test_X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import pickle\n",
    "# with open('feat.pkl','wb') as fout:\n",
    "#     pickle.dump([f_train_X,f_test_X],fout)\n",
    "# print('dump for xgb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def done\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "def cv_test(k_cnt=3, s_flag = False):\n",
    "    rnd = 42\n",
    "    if s_flag:\n",
    "        kf = StratifiedKFold(n_splits=k_cnt, shuffle=True, random_state=rnd)\n",
    "    else:\n",
    "        kf = KFold(n_splits=k_cnt, shuffle=True, random_state=rnd)\n",
    "    test_pred = None\n",
    "    weighted_test_pred = None\n",
    "    org_train_pred = None\n",
    "    avg_k_score = 0\n",
    "    reverse_score = 0\n",
    "    for train_index, test_index in kf.split(f_train_X,train_Y):\n",
    "        X_train, X_test = f_train_X[train_index], f_train_X[test_index]\n",
    "        y_train, y_test = train_Y[train_index], train_Y[test_index]\n",
    "        params = {\n",
    "                'colsample_bytree': 0.7,\n",
    "                'subsample': 0.8,\n",
    "                'eta': 0.04,\n",
    "                'max_depth': 3,\n",
    "                'eval_metric':'mlogloss',\n",
    "                'objective':'multi:softprob',\n",
    "                'num_class':3,\n",
    "                }\n",
    "        \n",
    "        # def mat\n",
    "        d_train = xgb.DMatrix(X_train, y_train)\n",
    "        d_valid = xgb.DMatrix(X_test, y_test)\n",
    "        d_test = xgb.DMatrix(f_test_X)\n",
    "        \n",
    "        watchlist = [(d_train, 'train'), (d_valid, 'valid')]\n",
    "        # train model\n",
    "        m = xgb.train(params, d_train, 2000, watchlist, \n",
    "                        early_stopping_rounds=50,\n",
    "                        verbose_eval=200)\n",
    "        \n",
    "        # get res\n",
    "        train_pred = m.predict(d_train)\n",
    "        valid_pred = m.predict(d_valid)\n",
    "        tmp_train_pred = m.predict(xgb.DMatrix(f_train_X))\n",
    "        \n",
    "        # cal score\n",
    "        train_score = log_loss(y_train,train_pred)\n",
    "        valid_score = log_loss(y_test,valid_pred)\n",
    "        print('train log loss',train_score,'valid log loss',valid_score)\n",
    "        avg_k_score += valid_score\n",
    "        rev_valid_score = 1.0/valid_score # use for weighted\n",
    "        reverse_score += rev_valid_score # sum\n",
    "        print('rev',rev_valid_score)\n",
    "        \n",
    "        if test_pred is None:\n",
    "            test_pred = m.predict(d_test)\n",
    "            weighted_test_pred = test_pred*rev_valid_score\n",
    "            org_train_pred = tmp_train_pred\n",
    "        else:\n",
    "            curr_pred = m.predict(d_test)\n",
    "            test_pred += curr_pred\n",
    "            weighted_test_pred += curr_pred*rev_valid_score # fix bug here\n",
    "            org_train_pred += tmp_train_pred\n",
    "\n",
    "    # avg\n",
    "    test_pred = test_pred / k_cnt\n",
    "    test_pred = np.round(test_pred,4)\n",
    "    org_train_pred = org_train_pred / k_cnt\n",
    "    avg_k_score = avg_k_score/k_cnt\n",
    "\n",
    "    submiss=pd.read_csv(\"./input/sample_submission.csv\")\n",
    "    submiss['EAP']=test_pred[:,0]\n",
    "    submiss['HPL']=test_pred[:,1]\n",
    "    submiss['MWS']=test_pred[:,2]\n",
    "    submiss.to_csv(\"results/xgb_res_{}_{}.csv\".format(k_cnt, s_flag),index=False)\n",
    "    print(submiss.head(5))\n",
    "    print('--------------')\n",
    "    print(reverse_score)\n",
    "    # weigthed\n",
    "    submiss=pd.read_csv(\"./input/sample_submission.csv\")\n",
    "    weighted_test_pred = weighted_test_pred / reverse_score\n",
    "    weighted_test_pred = np.round(weighted_test_pred,4)\n",
    "    submiss['EAP']=weighted_test_pred[:,0]\n",
    "    submiss['HPL']=weighted_test_pred[:,1]\n",
    "    submiss['MWS']=weighted_test_pred[:,2]\n",
    "    submiss.to_csv(\"results/weighted_xgb_res_{}_{}.csv\".format(k_cnt, s_flag),index=False)\n",
    "    print(submiss.head(5))\n",
    "    print('---------------')\n",
    "    \n",
    "    # train log loss\n",
    "    print('local average valid loss',avg_k_score)\n",
    "    print('train log loss', log_loss(train_Y,org_train_pred))\n",
    "    \n",
    "print('def done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-mlogloss:1.05555\tvalid-mlogloss:1.05667\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 50 rounds.\n",
      "[200]\ttrain-mlogloss:0.237486\tvalid-mlogloss:0.296052\n",
      "[400]\ttrain-mlogloss:0.19417\tvalid-mlogloss:0.288306\n",
      "Stopping. Best iteration:\n",
      "[520]\ttrain-mlogloss:0.175343\tvalid-mlogloss:0.287182\n",
      "\n",
      "train log loss 0.168097513234 valid log loss 0.287383714854\n",
      "rev 3.47966829125\n",
      "[0]\ttrain-mlogloss:1.05544\tvalid-mlogloss:1.05635\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 50 rounds.\n",
      "[200]\ttrain-mlogloss:0.238274\tvalid-mlogloss:0.289088\n",
      "[400]\ttrain-mlogloss:0.195138\tvalid-mlogloss:0.283524\n",
      "Stopping. Best iteration:\n",
      "[455]\ttrain-mlogloss:0.185842\tvalid-mlogloss:0.283068\n",
      "\n",
      "train log loss 0.178215087923 valid log loss 0.28334943224\n",
      "rev 3.52921123609\n",
      "[0]\ttrain-mlogloss:1.05602\tvalid-mlogloss:1.05558\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 50 rounds.\n",
      "[200]\ttrain-mlogloss:0.245915\tvalid-mlogloss:0.259999\n",
      "[400]\ttrain-mlogloss:0.203005\tvalid-mlogloss:0.252095\n",
      "[600]\ttrain-mlogloss:0.172528\tvalid-mlogloss:0.250481\n",
      "Stopping. Best iteration:\n",
      "[603]\ttrain-mlogloss:0.172053\tvalid-mlogloss:0.250427\n",
      "\n",
      "train log loss 0.16569896777 valid log loss 0.250477676712\n",
      "rev 3.99237174795\n",
      "[0]\ttrain-mlogloss:1.0558\tvalid-mlogloss:1.05617\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 50 rounds.\n",
      "[200]\ttrain-mlogloss:0.241159\tvalid-mlogloss:0.277024\n",
      "[400]\ttrain-mlogloss:0.198512\tvalid-mlogloss:0.269619\n",
      "[600]\ttrain-mlogloss:0.168516\tvalid-mlogloss:0.267302\n",
      "Stopping. Best iteration:\n",
      "[663]\ttrain-mlogloss:0.160486\tvalid-mlogloss:0.266552\n",
      "\n",
      "train log loss 0.154432971953 valid log loss 0.266604286436\n",
      "rev 3.75087742725\n",
      "[0]\ttrain-mlogloss:1.05574\tvalid-mlogloss:1.05602\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 50 rounds.\n",
      "[200]\ttrain-mlogloss:0.241579\tvalid-mlogloss:0.278845\n",
      "[400]\ttrain-mlogloss:0.198933\tvalid-mlogloss:0.271557\n",
      "[600]\ttrain-mlogloss:0.168845\tvalid-mlogloss:0.270199\n",
      "Stopping. Best iteration:\n",
      "[558]\ttrain-mlogloss:0.174322\tvalid-mlogloss:0.270017\n",
      "\n",
      "train log loss 0.16777933291 valid log loss 0.270092359469\n",
      "rev 3.7024372032\n",
      "        id     EAP     HPL     MWS\n",
      "0  id02310  0.0095  0.0029  0.9876\n",
      "1  id24541  0.9991  0.0005  0.0004\n",
      "2  id00134  0.0024  0.9968  0.0008\n",
      "3  id27757  0.8155  0.1799  0.0046\n",
      "4  id04081  0.8150  0.1125  0.0725\n",
      "--------------\n",
      "18.4545659057\n",
      "        id     EAP     HPL     MWS\n",
      "0  id02310  0.0097  0.0029  0.9874\n",
      "1  id24541  0.9991  0.0005  0.0004\n",
      "2  id00134  0.0024  0.9968  0.0008\n",
      "3  id27757  0.8145  0.1808  0.0046\n",
      "4  id04081  0.8141  0.1131  0.0728\n",
      "---------------\n",
      "local average valid loss 0.271581493942\n",
      "train log loss 0.179907872891\n"
     ]
    }
   ],
   "source": [
    "cv_test(5, True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}