features/bigram_supervised_fasttext.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append('../')\n",
    "from utils import preprocess, Tokenizer4keras, create_fastText_model\n",
    "\n",
    "import numpy as np\n",
    "import h5py\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from collections import defaultdict\n",
    "import string\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "import keras\n",
    "\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "from keras.utils import to_categorical\n",
    "\n",
    "np.random.seed(1234)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('./../data/train_feature.csv')\n",
    "df_test = pd.read_csv('./../data/test_feature.csv')\n",
    "df_train_texts = df_train.text.values\n",
    "df_test_tests = df_test.text.values\n",
    "\n",
    "author2class = {'EAP': 0, 'HPL' : 1, 'MWS' : 2}\n",
    "class2author = ['EAP', 'HPL', 'MWS']\n",
    "y = np.array([author2class[a] for a in df_train.author])\n",
    "y = to_categorical(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# preprocessin parameters\n",
    "n_gram_max = 2\n",
    "min_count = 2\n",
    "maxlen = 256\n",
    "\n",
    "tokenizer = Tokenizer4keras(maxlen=maxlen, min_count=min_count, n_gram_max=n_gram_max, lower=False, single=False, add_ngram_first=True)\n",
    "x = tokenizer.fit_transform(df_texts=df_train_texts)\n",
    "x_test = tokenizer.transofrm(df_test_tests)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/5: #Trains: 15663, #Val: 3916 (15663, 256) (3916, 256) (8392, 256)\n",
      "Train on 15663 samples, validate on 3916 samples\n",
      "Epoch 1/60\n",
      "15663/15663 [==============================] - 16s - loss: 1.0761 - acc: 0.4048 - val_loss: 1.0558 - val_acc: 0.4081\n",
      "Epoch 2/60\n",
      "15663/15663 [==============================] - 14s - loss: 1.0003 - acc: 0.4995 - val_loss: 0.9563 - val_acc: 0.5403\n",
      "Epoch 3/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.8573 - acc: 0.7132 - val_loss: 0.8269 - val_acc: 0.6948\n",
      "Epoch 4/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.7130 - acc: 0.7993 - val_loss: 0.7223 - val_acc: 0.7451\n",
      "Epoch 5/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.5974 - acc: 0.8397 - val_loss: 0.6473 - val_acc: 0.7579\n",
      "Epoch 6/60\n",
      "15663/15663 [==============================] - 14s - loss: 0.5063 - acc: 0.8690 - val_loss: 0.5860 - val_acc: 0.7842\n",
      "Epoch 7/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.4321 - acc: 0.8883 - val_loss: 0.5396 - val_acc: 0.8118\n",
      "Epoch 8/60\n",
      "15663/15663 [==============================] - 16s - loss: 0.3708 - acc: 0.9107 - val_loss: 0.4990 - val_acc: 0.8174\n",
      "Epoch 9/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.3189 - acc: 0.9237 - val_loss: 0.4672 - val_acc: 0.8246\n",
      "Epoch 10/60\n",
      "15663/15663 [==============================] - 16s - loss: 0.2746 - acc: 0.9380 - val_loss: 0.4418 - val_acc: 0.8312\n",
      "Epoch 11/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.2365 - acc: 0.9496 - val_loss: 0.4190 - val_acc: 0.8470\n",
      "Epoch 12/60\n",
      "15663/15663 [==============================] - 16s - loss: 0.2044 - acc: 0.9580 - val_loss: 0.4040 - val_acc: 0.8435\n",
      "Epoch 13/60\n",
      "15663/15663 [==============================] - 16s - loss: 0.1767 - acc: 0.9642 - val_loss: 0.3861 - val_acc: 0.8567\n",
      "Epoch 14/60\n",
      "15663/15663 [==============================] - 16s - loss: 0.1530 - acc: 0.9698 - val_loss: 0.3747 - val_acc: 0.8573\n",
      "Epoch 15/60\n",
      "15663/15663 [==============================] - 17s - loss: 0.1324 - acc: 0.9755 - val_loss: 0.3653 - val_acc: 0.8606\n",
      "Epoch 16/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.1147 - acc: 0.9789 - val_loss: 0.3577 - val_acc: 0.8654\n",
      "Epoch 17/60\n",
      "15663/15663 [==============================] - 17s - loss: 0.0994 - acc: 0.9822 - val_loss: 0.3500 - val_acc: 0.8636\n",
      "Epoch 18/60\n",
      "15663/15663 [==============================] - 18s - loss: 0.0862 - acc: 0.9855 - val_loss: 0.3457 - val_acc: 0.8685\n",
      "Epoch 19/60\n",
      "15663/15663 [==============================] - 17s - loss: 0.0748 - acc: 0.9879 - val_loss: 0.3445 - val_acc: 0.8690\n",
      "Epoch 20/60\n",
      "15663/15663 [==============================] - 17s - loss: 0.0646 - acc: 0.9898 - val_loss: 0.3412 - val_acc: 0.8710\n",
      "Epoch 21/60\n",
      "15663/15663 [==============================] - 17s - loss: 0.0567 - acc: 0.9909 - val_loss: 0.3392 - val_acc: 0.8710\n",
      "Epoch 22/60\n",
      "15663/15663 [==============================] - 16s - loss: 0.0491 - acc: 0.9929 - val_loss: 0.3359 - val_acc: 0.8710\n",
      "Epoch 23/60\n",
      "15663/15663 [==============================] - 16s - loss: 0.0430 - acc: 0.9939 - val_loss: 0.3364 - val_acc: 0.8721\n",
      "Epoch 24/60\n",
      "15663/15663 [==============================] - 14s - loss: 0.0371 - acc: 0.9950 - val_loss: 0.3464 - val_acc: 0.8677\n",
      "Epoch 25/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.0326 - acc: 0.9960 - val_loss: 0.3464 - val_acc: 0.8675\n",
      "Epoch 26/60\n",
      "15663/15663 [==============================] - 14s - loss: 0.0286 - acc: 0.9961 - val_loss: 0.3434 - val_acc: 0.8687\n",
      "Epoch 27/60\n",
      "15663/15663 [==============================] - 14s - loss: 0.0250 - acc: 0.9971 - val_loss: 0.3483 - val_acc: 0.8703\n",
      "2016/3916 [==============>...............] - ETA: 0svalLoss: 0.3359284443872644\n",
      "7840/8392 [===========================>..] - ETA: 0s2/5: #Trains: 15663, #Val: 3916 (15663, 256) (3916, 256) (8392, 256)\n",
      "Train on 15663 samples, validate on 3916 samples\n",
      "Epoch 1/60\n",
      "15663/15663 [==============================] - 15s - loss: 1.0780 - acc: 0.4027 - val_loss: 1.0560 - val_acc: 0.4160\n",
      "Epoch 2/60\n",
      "15663/15663 [==============================] - 14s - loss: 1.0037 - acc: 0.4927 - val_loss: 0.9505 - val_acc: 0.5541\n",
      "Epoch 3/60\n",
      "15663/15663 [==============================] - 14s - loss: 0.8609 - acc: 0.6893 - val_loss: 0.8206 - val_acc: 0.6793\n",
      "Epoch 4/60\n",
      "15663/15663 [==============================] - 16s - loss: 0.7160 - acc: 0.7937 - val_loss: 0.7121 - val_acc: 0.7643\n",
      "Epoch 5/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.5988 - acc: 0.8394 - val_loss: 0.6309 - val_acc: 0.7924\n",
      "Epoch 6/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.5057 - acc: 0.8697 - val_loss: 0.5693 - val_acc: 0.8133\n",
      "Epoch 7/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.4302 - acc: 0.8939 - val_loss: 0.5234 - val_acc: 0.8210\n",
      "Epoch 8/60\n",
      "15663/15663 [==============================] - 14s - loss: 0.3680 - acc: 0.9102 - val_loss: 0.4872 - val_acc: 0.8297\n",
      "Epoch 9/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.3160 - acc: 0.9266 - val_loss: 0.4520 - val_acc: 0.8381\n",
      "Epoch 10/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.2713 - acc: 0.9383 - val_loss: 0.4335 - val_acc: 0.8394\n",
      "Epoch 11/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.2335 - acc: 0.9485 - val_loss: 0.4048 - val_acc: 0.8504\n",
      "Epoch 12/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.2010 - acc: 0.9570 - val_loss: 0.3903 - val_acc: 0.8516\n",
      "Epoch 13/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.1730 - acc: 0.9656 - val_loss: 0.3757 - val_acc: 0.8580\n",
      "Epoch 14/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.1492 - acc: 0.9702 - val_loss: 0.3639 - val_acc: 0.8608\n",
      "Epoch 15/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.1288 - acc: 0.9757 - val_loss: 0.3519 - val_acc: 0.8641\n",
      "Epoch 16/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.1110 - acc: 0.9798 - val_loss: 0.3487 - val_acc: 0.8634\n",
      "Epoch 17/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.0958 - acc: 0.9831 - val_loss: 0.3399 - val_acc: 0.8672\n",
      "Epoch 18/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.0828 - acc: 0.9853 - val_loss: 0.3385 - val_acc: 0.8664\n",
      "Epoch 19/60\n",
      "15663/15663 [==============================] - 16s - loss: 0.0714 - acc: 0.9877 - val_loss: 0.3330 - val_acc: 0.8685\n",
      "Epoch 20/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.0621 - acc: 0.9899 - val_loss: 0.3346 - val_acc: 0.8687\n",
      "Epoch 21/60\n",
      "15663/15663 [==============================] - 14s - loss: 0.0536 - acc: 0.9918 - val_loss: 0.3308 - val_acc: 0.8710\n",
      "Epoch 22/60\n",
      "15663/15663 [==============================] - 15s - loss: 0.0463 - acc: 0.9935 - val_loss: 0.3330 - val_acc: 0.8726\n",
      "Epoch 23/60\n",
      "15663/15663 [==============================] - 14s - loss: 0.0403 - acc: 0.9948 - val_loss: 0.3326 - val_acc: 0.8716\n",
      "Epoch 24/60\n",
      "15663/15663 [==============================] - 14s - loss: 0.0350 - acc: 0.9952 - val_loss: 0.3346 - val_acc: 0.8731\n",
      "Epoch 25/60\n",
      "15663/15663 [==============================] - 14s - loss: 0.0301 - acc: 0.9959 - val_loss: 0.3364 - val_acc: 0.8733\n",
      "Epoch 26/60\n",
      "15663/15663 [==============================] - 14s - loss: 0.0264 - acc: 0.9965 - val_loss: 0.3455 - val_acc: 0.8723\n",
      "2592/3916 [==================>...........] - ETA: 0svalLoss: 0.33337840733017615\n",
      "6816/8392 [=======================>......] - ETA: 0s3/5: #Trains: 15663, #Val: 3916 (15663, 256) (3916, 256) (8392, 256)\n",
      "Train on 15663 samples, validate on 3916 samples\n",
      "Epoch 1/60\n",
      "15663/15663 [==============================] - 14s - loss: 1.0773 - acc: 0.4023 - val_loss: 1.0544 - val_acc: 0.4142\n",
      "Epoch 2/60\n",
      "15663/15663 [==============================] - 14s - loss: 1.0042 - acc: 0.4834 - val_loss: 0.9499 - val_acc: 0.5575\n",
      "Epoch 3/60\n",
      "15663/15663 [==============================] - 14s - loss: 0.8602 - acc: 0.7096 - val_loss: 0.8178 - val_acc: 0.7155\n",
      "Epoch 4/60\n",
      "15663/15663 [==============================] - 14s - loss: 0.7129 - acc: 0.8007 - val_loss: 0.7106 - val_acc: 0.7638\n",
      "Epoch 5/60\n",
      "15663/15663 [==============================] - 14s - loss: 0.5944 - acc: 0.8422 - val_loss: 0.6314 - val_acc: 0.7957\n",
      "Epoch 6/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.5010 - acc: 0.8703 - val_loss: 0.5711 - val_acc: 0.7993\n",
      "Epoch 7/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.4259 - acc: 0.8911 - val_loss: 0.5268 - val_acc: 0.8092\n",
      "Epoch 8/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.3635 - acc: 0.9109 - val_loss: 0.4841 - val_acc: 0.8304\n",
      "Epoch 9/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.3108 - acc: 0.9260 - val_loss: 0.4541 - val_acc: 0.8391\n",
      "Epoch 10/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.2661 - acc: 0.9399 - val_loss: 0.4297 - val_acc: 0.8445\n",
      "Epoch 11/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.2282 - acc: 0.9497 - val_loss: 0.4070 - val_acc: 0.8516\n",
      "Epoch 12/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.1961 - acc: 0.9589 - val_loss: 0.3997 - val_acc: 0.8478\n",
      "Epoch 13/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.1683 - acc: 0.9651 - val_loss: 0.3768 - val_acc: 0.8527\n",
      "Epoch 14/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.1452 - acc: 0.9713 - val_loss: 0.3672 - val_acc: 0.8580\n",
      "Epoch 15/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.1249 - acc: 0.9761 - val_loss: 0.3574 - val_acc: 0.8641\n",
      "Epoch 16/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.1076 - acc: 0.9807 - val_loss: 0.3491 - val_acc: 0.8624\n",
      "Epoch 17/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.0923 - acc: 0.9837 - val_loss: 0.3424 - val_acc: 0.8690\n",
      "Epoch 18/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.0795 - acc: 0.9869 - val_loss: 0.3400 - val_acc: 0.8677\n",
      "Epoch 19/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.0687 - acc: 0.9890 - val_loss: 0.3372 - val_acc: 0.8708\n",
      "Epoch 20/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.0594 - acc: 0.9914 - val_loss: 0.3354 - val_acc: 0.8718\n",
      "Epoch 21/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.0511 - acc: 0.9922 - val_loss: 0.3368 - val_acc: 0.8693\n",
      "Epoch 22/60\n",
      "15663/15663 [==============================] - 11s - loss: 0.0442 - acc: 0.9932 - val_loss: 0.3398 - val_acc: 0.8672\n",
      "Epoch 23/60\n",
      "15663/15663 [==============================] - 11s - loss: 0.0383 - acc: 0.9945 - val_loss: 0.3368 - val_acc: 0.8713\n",
      "Epoch 24/60\n",
      "15663/15663 [==============================] - 11s - loss: 0.0333 - acc: 0.9951 - val_loss: 0.3402 - val_acc: 0.8693\n",
      "Epoch 25/60\n",
      "15663/15663 [==============================] - 11s - loss: 0.0287 - acc: 0.9962 - val_loss: 0.3377 - val_acc: 0.8685\n",
      "3456/3916 [=========================>....] - ETA: 0svalLoss: 0.33404850955437365\n",
      "6624/8392 [======================>.......] - ETA: 0s4/5: #Trains: 15663, #Val: 3916 (15663, 256) (3916, 256) (8392, 256)\n",
      "Train on 15663 samples, validate on 3916 samples\n",
      "Epoch 1/60\n",
      "15663/15663 [==============================] - 12s - loss: 1.0777 - acc: 0.4031 - val_loss: 1.0633 - val_acc: 0.4063\n",
      "Epoch 2/60\n",
      "15663/15663 [==============================] - 12s - loss: 1.0083 - acc: 0.4935 - val_loss: 0.9745 - val_acc: 0.5051\n",
      "Epoch 3/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.8803 - acc: 0.6598 - val_loss: 0.8668 - val_acc: 0.6525\n",
      "Epoch 4/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.7534 - acc: 0.7689 - val_loss: 0.7721 - val_acc: 0.7337\n",
      "Epoch 5/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.6432 - acc: 0.8203 - val_loss: 0.6939 - val_acc: 0.7651\n",
      "Epoch 6/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.5493 - acc: 0.8529 - val_loss: 0.6338 - val_acc: 0.8011\n",
      "Epoch 7/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.4715 - acc: 0.8805 - val_loss: 0.5785 - val_acc: 0.7978\n",
      "Epoch 8/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.4041 - acc: 0.9001 - val_loss: 0.5360 - val_acc: 0.8233\n",
      "Epoch 9/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.3481 - acc: 0.9173 - val_loss: 0.4989 - val_acc: 0.8218\n",
      "Epoch 10/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.2995 - acc: 0.9319 - val_loss: 0.4689 - val_acc: 0.8355\n",
      "Epoch 11/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.2589 - acc: 0.9437 - val_loss: 0.4441 - val_acc: 0.8419\n",
      "Epoch 12/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.2229 - acc: 0.9545 - val_loss: 0.4239 - val_acc: 0.8417\n",
      "Epoch 13/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.1927 - acc: 0.9612 - val_loss: 0.4062 - val_acc: 0.8506\n",
      "Epoch 14/60\n",
      "15663/15663 [==============================] - 13s - loss: 0.1663 - acc: 0.9681 - val_loss: 0.3946 - val_acc: 0.8516\n",
      "Epoch 15/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.1437 - acc: 0.9729 - val_loss: 0.3875 - val_acc: 0.8478\n",
      "Epoch 16/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.1245 - acc: 0.9777 - val_loss: 0.3725 - val_acc: 0.8616\n",
      "Epoch 17/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.1075 - acc: 0.9817 - val_loss: 0.3685 - val_acc: 0.8626\n",
      "Epoch 18/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.0936 - acc: 0.9843 - val_loss: 0.3610 - val_acc: 0.8626\n",
      "Epoch 19/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.0807 - acc: 0.9870 - val_loss: 0.3565 - val_acc: 0.8652\n",
      "Epoch 20/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.0702 - acc: 0.9891 - val_loss: 0.3557 - val_acc: 0.8693\n",
      "Epoch 21/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.0609 - acc: 0.9912 - val_loss: 0.3545 - val_acc: 0.8644\n",
      "Epoch 22/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.0532 - acc: 0.9926 - val_loss: 0.3561 - val_acc: 0.8687\n",
      "Epoch 23/60\n",
      "15663/15663 [==============================] - 11s - loss: 0.0461 - acc: 0.9941 - val_loss: 0.3580 - val_acc: 0.8626\n",
      "Epoch 24/60\n",
      "15663/15663 [==============================] - 12s - loss: 0.0402 - acc: 0.9948 - val_loss: 0.3565 - val_acc: 0.8664\n",
      "Epoch 25/60\n",
      "15663/15663 [==============================] - 11s - loss: 0.0351 - acc: 0.9955 - val_loss: 0.3644 - val_acc: 0.8652\n",
      "Epoch 26/60\n",
      "15663/15663 [==============================] - 11s - loss: 0.0308 - acc: 0.9964 - val_loss: 0.3646 - val_acc: 0.8659\n",
      "3264/3916 [========================>.....] - ETA: 0svalLoss: 0.33915055815879896\n",
      "6496/8392 [======================>.......] - ETA: 0s5/5: #Trains: 15664, #Val: 3915 (15664, 256) (3915, 256) (8392, 256)\n",
      "Train on 15664 samples, validate on 3915 samples\n",
      "Epoch 1/60\n",
      "15664/15664 [==============================] - 13s - loss: 1.0766 - acc: 0.4037 - val_loss: 1.0571 - val_acc: 0.4059\n",
      "Epoch 2/60\n",
      "15664/15664 [==============================] - 13s - loss: 1.0020 - acc: 0.4839 - val_loss: 0.9595 - val_acc: 0.5504\n",
      "Epoch 3/60\n",
      "15664/15664 [==============================] - 13s - loss: 0.8601 - acc: 0.7071 - val_loss: 0.8253 - val_acc: 0.7185\n",
      "Epoch 4/60\n",
      "15664/15664 [==============================] - 13s - loss: 0.7143 - acc: 0.7971 - val_loss: 0.7163 - val_acc: 0.7747\n",
      "Epoch 5/60\n",
      "15664/15664 [==============================] - 13s - loss: 0.5969 - acc: 0.8415 - val_loss: 0.6391 - val_acc: 0.7949\n",
      "Epoch 6/60\n",
      "15664/15664 [==============================] - 13s - loss: 0.5046 - acc: 0.8657 - val_loss: 0.5764 - val_acc: 0.8112\n",
      "Epoch 7/60\n",
      "15664/15664 [==============================] - 13s - loss: 0.4300 - acc: 0.8901 - val_loss: 0.5306 - val_acc: 0.8230\n",
      "Epoch 8/60\n",
      "15664/15664 [==============================] - 13s - loss: 0.3680 - acc: 0.9086 - val_loss: 0.4919 - val_acc: 0.8273\n",
      "Epoch 9/60\n",
      "15664/15664 [==============================] - 13s - loss: 0.3164 - acc: 0.9237 - val_loss: 0.4615 - val_acc: 0.8363\n",
      "Epoch 10/60\n",
      "15664/15664 [==============================] - 13s - loss: 0.2717 - acc: 0.9365 - val_loss: 0.4361 - val_acc: 0.8391\n",
      "Epoch 11/60\n",
      "15664/15664 [==============================] - 13s - loss: 0.2345 - acc: 0.9489 - val_loss: 0.4156 - val_acc: 0.8455\n",
      "Epoch 12/60\n",
      "15664/15664 [==============================] - 13s - loss: 0.2021 - acc: 0.9570 - val_loss: 0.4018 - val_acc: 0.8480\n",
      "Epoch 13/60\n",
      "15664/15664 [==============================] - 13s - loss: 0.1748 - acc: 0.9646 - val_loss: 0.3861 - val_acc: 0.8524\n",
      "Epoch 14/60\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15664/15664 [==============================] - 13s - loss: 0.1506 - acc: 0.9706 - val_loss: 0.3751 - val_acc: 0.8534\n",
      "Epoch 15/60\n",
      "15664/15664 [==============================] - 13s - loss: 0.1300 - acc: 0.9774 - val_loss: 0.3686 - val_acc: 0.8554\n",
      "Epoch 16/60\n",
      "15664/15664 [==============================] - 13s - loss: 0.1129 - acc: 0.9803 - val_loss: 0.3583 - val_acc: 0.8610\n",
      "Epoch 17/60\n",
      "15664/15664 [==============================] - 13s - loss: 0.0977 - acc: 0.9830 - val_loss: 0.3532 - val_acc: 0.8616\n",
      "Epoch 18/60\n",
      "15664/15664 [==============================] - 13s - loss: 0.0842 - acc: 0.9860 - val_loss: 0.3533 - val_acc: 0.8600\n",
      "Epoch 19/60\n",
      "15664/15664 [==============================] - 12s - loss: 0.0729 - acc: 0.9884 - val_loss: 0.3601 - val_acc: 0.8590\n",
      "Epoch 20/60\n",
      "15664/15664 [==============================] - 12s - loss: 0.0634 - acc: 0.9899 - val_loss: 0.3458 - val_acc: 0.8633\n",
      "Epoch 21/60\n",
      "15664/15664 [==============================] - 13s - loss: 0.0550 - acc: 0.9911 - val_loss: 0.3481 - val_acc: 0.8656\n",
      "Epoch 22/60\n",
      "15664/15664 [==============================] - 12s - loss: 0.0480 - acc: 0.9927 - val_loss: 0.3459 - val_acc: 0.8679\n",
      "Epoch 23/60\n",
      "15664/15664 [==============================] - 12s - loss: 0.0416 - acc: 0.9934 - val_loss: 0.3474 - val_acc: 0.8682\n",
      "Epoch 24/60\n",
      "15664/15664 [==============================] - 12s - loss: 0.0364 - acc: 0.9947 - val_loss: 0.3490 - val_acc: 0.8690\n",
      "Epoch 25/60\n",
      "15664/15664 [==============================] - 12s - loss: 0.0316 - acc: 0.9953 - val_loss: 0.3546 - val_acc: 0.8677\n",
      "3200/3915 [=======================>......] - ETA: 0svalLoss: 0.34047266946487104\n",
      "6336/8392 [=====================>........] - ETA: 0s"
     ]
    }
   ],
   "source": [
    "input_dim = np.max(x) + 1\n",
    "\n",
    "# for next training\n",
    "predict_prob_features = np.zeros((len(df_train), 3))\n",
    "predict_prob_features_test = np.zeros((len(df_test), 3))\n",
    "\n",
    "# training parameters\n",
    "seed = 7\n",
    "num_split = 5\n",
    "epochs = 60\n",
    "\n",
    "ite = 0\n",
    "sum_loss = 0.\n",
    "losses = []\n",
    "\n",
    "kf = KFold(n_splits=num_split, random_state=seed, shuffle=True)\n",
    "for train_index, val_index in kf.split(x):\n",
    "    ite += 1\n",
    "    print('{}/{}: #Trains: {}, #Val: {}'.format(ite, num_split, len(train_index), len(val_index)), end=' ')\n",
    "    \n",
    "    x_train = x[train_index]\n",
    "    x_val = x[val_index]\n",
    "\n",
    "    y_train, y_val = y[train_index], y[val_index]\n",
    "\n",
    "    print(x_train.shape, x_val.shape, x_test.shape)\n",
    "    \n",
    "    model = create_fastText_model(input_dim, embedding_dim=10, optimizer='adam')\n",
    "    \n",
    "    checkpointer = ModelCheckpoint(filepath='./../fasttext_weights/weights_bi.hdf5', verbose=0, save_best_only=True)\n",
    "\n",
    "    hist = model.fit(x_train, y_train,\n",
    "                     batch_size=16,\n",
    "                     validation_data=(x_val, y_val),\n",
    "                     epochs=epochs,\n",
    "                     callbacks=[EarlyStopping(patience=4, monitor='val_loss'), checkpointer]\n",
    "                    )\n",
    "\n",
    "\n",
    "    # load best weights\n",
    "    model.load_weights('./../fasttext_weights/weights_bi.hdf5')\n",
    "    y_pred = model.predict_proba(x_val)\n",
    "    l = log_loss(y_pred=y_pred, y_true=np.nonzero(y_val)[1])\n",
    "    losses.append(l)\n",
    "    sum_loss += l\n",
    "    \n",
    "    print('valLoss: {}'.format(sum_loss/ite))\n",
    "\n",
    "    # save features\n",
    "    predict_prob_features[val_index] = y_pred\n",
    "    predict_prob_features_test += model.predict_proba(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0.33592844438726438,\n",
       "  0.33082837027308792,\n",
       "  0.33538871400276848,\n",
       "  0.3544567039720749,\n",
       "  0.34576111468915938],\n",
       " 0.34047266946487104)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "losses, np.mean(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for a, c in author2class.items():\n",
    "    df_train['{}_fasttext_bigram'.format(a)] = predict_prob_features[:, c]\n",
    "    df_test['{}_fasttext_bigram'.format(a)] = predict_prob_features_test[:, c]/num_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_train.to_csv('./../data/train_feature.csv', index=False)\n",
    "df_test.to_csv('./../data/test_feature.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}