PTT board classification attention-RNN model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {}
     ]
    },
    "colab_type": "code",
    "id": "D72meokFwX7J",
    "outputId": "420d76a4-edf0-492d-9ee9-07b6933c685b"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "190190\n",
      "0 的 453107 83139\n",
      "1 我 184079 56942\n",
      "2 是 153676 65403\n",
      "3 了 126424 57126\n",
      "4 有 104298 55936\n",
      "5 在 94438 51777\n",
      "6 也 85862 48035\n",
      "7 都 73951 44112\n",
      "8 就 70596 41607\n",
      "9 不 64110 40699\n",
      "10 很 53089 33219\n",
      "11 會 51420 33428\n",
      "12 他 49080 19704\n",
      "13 跟 46775 29688\n",
      "14 你 46472 20085\n",
      "15 想 45924 32358\n",
      "16 說 45526 28978\n",
      "17 看 45432 27455\n",
      "18 可以 45145 29301\n",
      "19 她 43894 14221\n",
      "1000 中間 1187 1135\n",
      "2000 f 569 453\n",
      "3000 實在太 343 338\n",
      "4000 作畫 238 182\n",
      "5000 六日 181 164\n",
      "6000 力晶 144 102\n",
      "7000 給些 117 117\n",
      "8000 一棟 98 91\n",
      "9000 免得 83 83\n",
      "10000 善於 73 69\n",
      "11000 隨身 64 48\n",
      "12000 超時空 57 43\n",
      "13000 數值 51 43\n",
      "14000 罪名 46 43\n",
      "15000 spec 42 37\n",
      "16000 滷味 38 38\n",
      "17000 有異 35 35\n",
      "18000 z017da 32 32\n",
      "19000 組員 30 20\n",
      "20000 電影界 28 27\n",
      "21000 滋潤 26 24\n",
      "22000 邪神 24 22\n",
      "23000 作對 22 20\n",
      "24000 尤里 21 11\n",
      "25000 必經 20 20\n",
      "26000 抓起 19 19\n",
      "27000 喇機 18 14\n",
      "28000 淪落 17 17\n",
      "29000 找下 16 16\n",
      "30000 龍龍 15 15\n",
      "31000 九淺 14 8\n",
      "32000 焊接 14 8\n",
      "33000 席格 13 9\n",
      "34000 家長會 13 13\n",
      "35000 2510p 12 6\n",
      "36000 自吹 12 2\n",
      "37000 大嘴巴 11 11\n",
      "38000 帶你去 11 11\n",
      "39000 交不 10 9\n",
      "40000 街機 10 8\n",
      "41000 山盟海誓 9 7\n",
      "42000 點整 9 3\n",
      "43000 貞潔 9 8\n",
      "44000 夫妻間 8 8\n",
      "45000 本會 8 8\n",
      "46000 颯爽 8 8\n",
      "47000 貶值 8 7\n",
      "48000 完賽 7 7\n",
      "49000 不算數 7 7\n",
      "50000 烹煮 7 7\n",
      "51000 陸委會 7 5\n",
      "52000 外片 7 5\n",
      "53000 瑟雷 6 3\n",
      "54000 張家 6 5\n",
      "55000 去勢 6 5\n",
      "56000 壓好 6 6\n",
      "57000 舞風 6 4\n",
      "58000 分獲 6 6\n",
      "59000 遲交 5 5\n",
      "60000 香篇 5 5\n",
      "61000 鼓山 5 4\n",
      "62000 小池 5 4\n",
      "63000 揭開序幕 5 5\n",
      "64000 屏南 5 5\n",
      "65000 手起刀落 5 3\n",
      "66000 fatima 4 4\n",
      "67000 傳送點 4 4\n",
      "68000 巨巨有 4 4\n",
      "69000 全砍 4 4\n",
      "70000 黛比 4 2\n",
      "71000 與僅 4 4\n",
      "72000 區還 4 4\n",
      "73000 吹哨子 4 2\n",
      "74000 光和愛花 4 2\n",
      "75000 快查 4 4\n",
      "76000 點徵人 4 4\n",
      "77000 海荷 4 4\n",
      "78000 那別 3 3\n",
      "79000 赫茲 3 3\n",
      "80000 展奪獎 3 3\n",
      "81000 未可知 3 3\n",
      "82000 保不住 3 3\n",
      "83000 blackphantom 3 3\n",
      "84000 齒痕 3 3\n",
      "85000 感壓過 3 3\n",
      "86000 勒著 3 3\n",
      "87000 尚餘 3 3\n",
      "88000 廢水處理 3 3\n",
      "89000 秋雨 3 3\n",
      "90000 溫婉 3 3\n",
      "91000 尤金尼 3 3\n",
      "92000 清好 3 3\n",
      "93000 遊吃個 3 3\n",
      "94000 明文禁止 3 3\n",
      "95000 星之琴 3 3\n",
      "96000 找王 3 3\n",
      "97000 mlcc 3 2\n",
      "98000 歐洛沃 2 2\n",
      "99000 奇數 2 1\n",
      "100000 為工 2 2\n",
      "101000 rumor 2 2\n",
      "102000 樹狀 2 2\n",
      "103000 安濟 2 2\n",
      "104000 tbt 2 2\n",
      "105000 瑟雷西 2 2\n",
      "106000 豬朋狗友 2 2\n",
      "107000 安還衛 2 2\n",
      "108000 kakashou 2 2\n",
      "109000 城時 2 2\n",
      "110000 fiennes 2 2\n",
      "111000 南勢角 2 2\n",
      "112000 apple751220 2 2\n",
      "113000 出診 2 2\n",
      "114000 再順 2 2\n",
      "115000 太夯 2 2\n",
      "116000 低壓 2 2\n",
      "117000 tyrese 2 2\n",
      "118000 輸配 2 2\n",
      "119000 當從者 2 2\n",
      "120000 地鐵線 2 2\n",
      "121000 開太 2 2\n",
      "122000 寶二 2 2\n",
      "123000 華語音樂 2 2\n",
      "124000 ortiz 2 2\n",
      "125000 兩人加 2 2\n",
      "126000 連打個 2 2\n",
      "127000 女甲 2 2\n",
      "128000 emo 2 2\n",
      "129000 網咖讓 2 2\n",
      "130000 在籍 2 2\n",
      "131000 0530 2 2\n",
      "132000 kidlin1990 2 2\n",
      "133000 隆重 2 2\n",
      "134000 4cd 2 2\n",
      "135000 將之風 1 1\n",
      "136000 戒塔 1 1\n",
      "137000 其盛 1 1\n",
      "138000 jasonkobe81 1 1\n",
      "139000 快長 1 1\n",
      "140000 玩互 1 1\n",
      "141000 看得會 1 1\n",
      "142000 honyihon 1 1\n",
      "143000 會滴個 1 1\n",
      "144000 沒掉 1 1\n",
      "145000 家收 1 1\n",
      "146000 閉路電視 1 1\n",
      "147000 爽發 1 1\n",
      "148000 抽簽 1 1\n",
      "149000 linmeng 1 1\n",
      "150000 幫班普 1 1\n",
      "151000 完喝 1 1\n",
      "152000 任合 1 1\n",
      "153000 招文 1 1\n",
      "154000 尼娘 1 1\n",
      "155000 吃大虧 1 1\n",
      "156000 ventus0707 1 1\n",
      "157000 寡然 1 1\n",
      "158000 快督 1 1\n",
      "159000 想久 1 1\n",
      "160000 叫度 1 1\n",
      "161000 引發出 1 1\n",
      "162000 宵小 1 1\n",
      "163000 mymountain 1 1\n",
      "164000 可作 1 1\n",
      "165000 男亮 1 1\n",
      "166000 duration 1 1\n",
      "167000 3338 1 1\n",
      "168000 再邊 1 1\n",
      "169000 白武 1 1\n",
      "170000 高寡 1 1\n",
      "171000 雅活 1 1\n",
      "172000 foreign 1 1\n",
      "173000 科陳恆貞 1 1\n",
      "174000 翻路 1 1\n",
      "175000 qaqqqqqqqq 1 1\n",
      "176000 960k 1 1\n",
      "177000 普下妹 1 1\n",
      "178000 寄個帳 1 1\n",
      "179000 工或 1 1\n",
      "180000 常不讀 1 1\n",
      "181000 多賴 1 1\n",
      "182000 junk 1 1\n",
      "183000 無義 1 1\n",
      "184000 跟槽點 1 1\n",
      "185000 用迪卡 1 1\n",
      "186000 marcela 1 1\n",
      "187000 kiske011 1 1\n",
      "188000 九命怪貓 1 1\n",
      "189000 5925 1 1\n",
      "190000 看安茲 1 1\n"
     ]
    }
   ],
   "source": [
    "# Classify PTT posts to broad by attention RNN model.\n",
    "import pandas as pd\n",
    "import time\n",
    "import csv\n",
    "import os\n",
    "import types \n",
    "\n",
    "#Initialize for files name and path.\n",
    "base_dir = 'C:/Users/User/Raw data/PTT'\n",
    "totalboard_name = 'TJ-BG-AT-SX-CC-MV'\n",
    "board_name = ['Tech_job','Boy-Girl','AllTogether','sex','C_Chat','movie']\n",
    "board_dict_index = {'Tech_job':0,'Boy-Girl':1,'AllTogether':2,'sex':3,'C_Chat':4,'movie':5}\n",
    "max_word_length = 256\n",
    "min_word_length = 100\n",
    "label_newnum = len(board_name)\n",
    "\n",
    "new_content_dir = []\n",
    "totalboard_dir = os.path.join(base_dir, totalboard_name)\n",
    "if not os.path.exists(totalboard_dir):\n",
    "    os.makedirs(totalboard_dir)\n",
    "for name_ind in range(len(board_name)):\n",
    "    # New data path\n",
    "    new_content_dir.append(os.path.join(totalboard_dir, board_name[name_ind]+'_content'))\n",
    "    if not os.path.exists(new_content_dir[name_ind]):\n",
    "        os.makedirs(new_content_dir[name_ind])\n",
    "\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "import jieba\n",
    "\n",
    "text_content = []\n",
    "text_label = []\n",
    "\n",
    "#dfs_total = pd.read_csv(os.path.join(totalboard_dir, totalboard_name + '-test.csv'))\n",
    "dfs_total = pd.read_csv(os.path.join(totalboard_dir, totalboard_name + '.csv'))\n",
    "len_total = len(dfs_total)\n",
    "\n",
    "for dfs_index in range(len_total):\n",
    "    read_index = int(dfs_total.iloc[dfs_index, 0])\n",
    "    type_name = str(dfs_total.iloc[dfs_index, 1])\n",
    "    word_length = int(dfs_total.iloc[dfs_index, 5])\n",
    "    title_name = str(dfs_total.iloc[dfs_index, 9])\n",
    "    this_board_name = str(dfs_total.iloc[dfs_index, 10])\n",
    "    \n",
    "    text = ''\n",
    "    with open(os.path.join(new_content_dir[board_dict_index[this_board_name]], str(read_index) + '.csv'),\n",
    "              'r', encoding = 'utf-8-sig') as file:\n",
    "        csvCursor = csv.reader(file)\n",
    "        for rows in csvCursor:\n",
    "            for row in rows:\n",
    "                # Read content and remove empty.\n",
    "                text = text + row\n",
    "    \n",
    "    text_content.append(text)\n",
    "    text_label.append(board_dict_index[this_board_name])\n",
    "    file.close()\n",
    "\n",
    "words_limit = 40000\n",
    "tokenizer = Tokenizer(num_words=words_limit, \n",
    "                      filters='!\"#$%&()*+,-./:;=?@[\\\\]^_`{|}~\\t\\n　，。！：；、？﹝﹞「」『』（）｛｝［］【】《》“”‘’＼｜〝〞‵′＋－＊／＝≦≧＿＠＃＄％︿＆～§◎．※ㄧ↔│○●☆★◇◆□■▽▼△▲㊣⊙⊕ˍ…﹌﹋﹎﹍﹉﹊‥–↑↓←→↖↗↙↘∥∕℅≒≡∩∪∞￣＿◤◥◣◢∵∴〒⊥∠⊿┼┴┬┤├▔─│▕┌┐└┘╭╮╰╯═╞╪╡╔╦╗╠╬╣╚╩╝╒╤╕╘╧╛╓╥╖╟╫╢╙╨╜║▓╱╲╳▁▂▄▅▆▇█▏▎▍▌▋▊▉▁▔', \n",
    "                      split=\" \")\n",
    "tokenizer.fit_on_texts(text_content)\n",
    "vocab = tokenizer.word_index\n",
    "print(len(vocab))\n",
    "\n",
    "vocab_counts = tokenizer.word_counts\n",
    "vocab_docs = tokenizer.word_docs\n",
    "vocab_index = 0\n",
    "for vocab_word in vocab:\n",
    "    if vocab_index % 1000 == 0 or vocab_index < 20:\n",
    "        print(vocab_index, vocab_word, vocab_counts[vocab_word], vocab_docs[vocab_word])\n",
    "    vocab_index += 1\n",
    "\n",
    "from keras.utils import np_utils\n",
    "x_train, x_test, y_train, y_test = train_test_split(text_content, text_label,\n",
    "                                                    test_size=0.2, random_state=9487)\n",
    "\n",
    "y_train = np_utils.to_categorical(y_train, label_newnum)\n",
    "y_test = np_utils.to_categorical(y_test, label_newnum)\n",
    "\n",
    "# 将每个词用词典中的数值代替\n",
    "x_train_word_ids = tokenizer.texts_to_sequences(x_train)\n",
    "x_test_word_ids = tokenizer.texts_to_sequences(x_test)\n",
    "# 序列模式\n",
    "x_train = pad_sequences(x_train_word_ids, maxlen=max_word_length)\n",
    "x_test = pad_sequences(x_test_word_ids, maxlen=max_word_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_index = 0\n",
    "for vocab_word in vocab:\n",
    "    if len(vocab_word)>2:\n",
    "        print(vocab_index, vocab_word, vocab_counts[vocab_word], vocab_docs[vocab_word])\n",
    "    vocab_index += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 1343,
     "output_extras": [
      {}
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9160,
     "status": "ok",
     "timestamp": 1517421499174,
     "user": {
      "displayName": "張維尼",
      "photoUrl": "//lh3.googleusercontent.com/-c5EqT2ZqGcE/AAAAAAAAAAI/AAAAAAAACAg/TmR2MAQg0Nc/s50-c-k-no/photo.jpg",
      "userId": "111480244562734483115"
     },
     "user_tz": -480
    },
    "id": "-iUlULwfCdGC",
    "outputId": "3a3b3192-bbe8-4534-81d5-53f2e05b9566",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "embedding_2 (Embedding)      (None, 256, 300)          12000300  \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 256, 300)          405900    \n",
      "_________________________________________________________________\n",
      "attention_1 (Attention)      (None, 300)               90000     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 60)                18060     \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 60)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 6)                 366       \n",
      "=================================================================\n",
      "Total params: 12,514,626\n",
      "Trainable params: 12,514,626\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "#######\n",
      "epoch1\n",
      "#######\n",
      "Iteration  108  ( 9.91 %) Train Loss:  0.482399 ; Train accuracy:  0.859375 \t\t\n",
      "Iteration  216  ( 19.91 %) Train Loss:  0.308538 ; Train accuracy:  0.921875 \t\t\n",
      "Iteration  324  ( 29.91 %) Train Loss:  0.237581 ; Train accuracy:  0.953125 \t\t\n",
      "Iteration  432  ( 39.91 %) Train Loss:  0.178677 ; Train accuracy:  0.953125 \t\t\n",
      "Iteration  540  ( 49.91 %) Train Loss:  0.184624 ; Train accuracy:  0.96875 \t\t\n",
      "Iteration  648  ( 59.91 %) Train Loss:  0.173699 ; Train accuracy:  0.953125 \t\t\n",
      "Iteration  756  ( 69.91 %) Train Loss:  0.194349 ; Train accuracy:  0.953125 \t\t\n",
      "Iteration  864  ( 79.91 %) Train Loss:  0.253613 ; Train accuracy:  0.9375 \t\t\n",
      "Iteration  972  ( 89.91 %) Train Loss:  0.475958 ; Train accuracy:  0.90625 \t\t\n",
      "Iteration  1080  ( 99.91 %) Train Loss:  0.161294 ; Train accuracy:  0.96875 \t\t\n",
      "Train Loss:  0.161294 ; Train accuracy:  0.96875\n",
      "17280/17280 [==============================] - 26s    \n",
      "Test Loss:  0.16837057107 : Test accuracy:  0.960127314815\n",
      "confusions [prediction\\True]\n",
      "[[2771   28    0    5   11    3]\n",
      " [  20 2749   12  100   10   13]\n",
      " [   8   48 2813    2    6   21]\n",
      " [  13  119    9 2726   23   26]\n",
      " [   7   18    2   37 2750   64]\n",
      " [   2    9    3    8   62 2782]]\n",
      "Save best score!! 0.960127314815\n",
      "Elapsed time in epoch 1: 510.22405219078064 [s]\n",
      "Saved model 1!\n",
      "\n",
      "#######\n",
      "epoch2\n",
      "#######\n",
      "Iteration  108  ( 9.91 %) Train Loss:  0.0907065 ; Train accuracy:  0.96875 \t\t\n",
      "Iteration  216  ( 19.91 %) Train Loss:  0.165466 ; Train accuracy:  0.96875 \t\t\n",
      "Iteration  324  ( 29.91 %) Train Loss:  0.16625 ; Train accuracy:  0.953125 \t\t\n",
      "Iteration  432  ( 39.91 %) Train Loss:  0.0594629 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  540  ( 49.91 %) Train Loss:  0.229502 ; Train accuracy:  0.96875 \t\t\n",
      "Iteration  648  ( 59.91 %) Train Loss:  0.183964 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  756  ( 69.91 %) Train Loss:  0.107057 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  864  ( 79.91 %) Train Loss:  0.147577 ; Train accuracy:  0.96875 \t\t\n",
      "Iteration  972  ( 89.91 %) Train Loss:  0.115008 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  1080  ( 99.91 %) Train Loss:  0.17428 ; Train accuracy:  0.953125 \t\t\n",
      "Train Loss:  0.17428 ; Train accuracy:  0.953125\n",
      "17280/17280 [==============================] - 26s    \n",
      "Test Loss:  0.159380994351 : Test accuracy:  0.962673611111\n",
      "confusions [prediction\\True]\n",
      "[[2759   24    2    5   20    8]\n",
      " [  13 2826    5   34   17    9]\n",
      " [   3   34 2838    7    9    7]\n",
      " [   3  136    3 2598   80   96]\n",
      " [   2    9    0    8 2822   37]\n",
      " [   1    6    3    0   64 2792]]\n",
      "Save best score!! 0.962673611111\n",
      "Elapsed time in epoch 2: 504.2317008972168 [s]\n",
      "Saved model 2!\n",
      "\n",
      "#######\n",
      "epoch3\n",
      "#######\n",
      "Iteration  108  ( 9.91 %) Train Loss:  0.0375864 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  216  ( 19.91 %) Train Loss:  0.0403329 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  324  ( 29.91 %) Train Loss:  0.0862252 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  432  ( 39.91 %) Train Loss:  0.156612 ; Train accuracy:  0.96875 \t\t\n",
      "Iteration  540  ( 49.91 %) Train Loss:  0.0364342 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  648  ( 59.91 %) Train Loss:  0.10189 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  756  ( 69.91 %) Train Loss:  0.0552433 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  864  ( 79.91 %) Train Loss:  0.0367188 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  972  ( 89.91 %) Train Loss:  0.118068 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  1080  ( 99.91 %) Train Loss:  0.063067 ; Train accuracy:  0.984375 \t\t\n",
      "Train Loss:  0.063067 ; Train accuracy:  0.984375\n",
      "17280/17280 [==============================] - 26s    \n",
      "Test Loss:  0.205508597471 : Test accuracy:  0.95625\n",
      "confusions [prediction\\True]\n",
      "[[2742   30   35    1    4    6]\n",
      " [  16 2772   46   52    4   14]\n",
      " [   3   10 2880    2    1    2]\n",
      " [  12   86   27 2760    6   25]\n",
      " [  20   35   80   36 2556  151]\n",
      " [   2    7   22    7   14 2814]]\n",
      "Count early stop!! 1\n",
      "Elapsed time in epoch 3: 504.13391160964966 [s]\n",
      "Saved model 3!\n",
      "\n",
      "#######\n",
      "epoch4\n",
      "#######\n",
      "Iteration  108  ( 9.91 %) Train Loss:  0.0459682 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  216  ( 19.91 %) Train Loss:  0.0502002 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  324  ( 29.91 %) Train Loss:  0.0348709 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  432  ( 39.91 %) Train Loss:  0.0341121 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  540  ( 49.91 %) Train Loss:  0.0412277 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  648  ( 59.91 %) Train Loss:  0.0225081 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  756  ( 69.91 %) Train Loss:  0.0949241 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  864  ( 79.91 %) Train Loss:  0.215879 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  972  ( 89.91 %) Train Loss:  0.111501 ; Train accuracy:  0.96875 \t\t\n",
      "Iteration  1080  ( 99.91 %) Train Loss:  0.0966315 ; Train accuracy:  1.0 \t\t\n",
      "Train Loss:  0.0966315 ; Train accuracy:  1.0\n",
      "17280/17280 [==============================] - 26s    \n",
      "Test Loss:  0.219810998964 : Test accuracy:  0.96724537037\n",
      "confusions [prediction\\True]\n",
      "[[2796    8    1    6    4    3]\n",
      " [  36 2780   18   56    8    6]\n",
      " [  11   26 2835    6    8   12]\n",
      " [  19   91    5 2778   14    9]\n",
      " [  23   18    0   50 2714   73]\n",
      " [   6    7    3   22   17 2811]]\n",
      "Save best score!! 0.96724537037\n",
      "Elapsed time in epoch 4: 502.741423368454 [s]\n",
      "Saved model 4!\n",
      "\n",
      "#######\n",
      "epoch5\n",
      "#######\n",
      "Iteration  108  ( 9.91 %) Train Loss:  0.196089 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  216  ( 19.91 %) Train Loss:  0.0303341 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  324  ( 29.91 %) Train Loss:  0.0282072 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  432  ( 39.91 %) Train Loss:  0.0607112 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  540  ( 49.91 %) Train Loss:  0.0958854 ; Train accuracy:  0.96875 \t\t\n",
      "Iteration  648  ( 59.91 %) Train Loss:  0.0448215 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  756  ( 69.91 %) Train Loss:  0.0442804 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  864  ( 79.91 %) Train Loss:  0.114669 ; Train accuracy:  0.96875 \t\t\n",
      "Iteration  972  ( 89.91 %) Train Loss:  0.196486 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  1080  ( 99.91 %) Train Loss:  0.0565905 ; Train accuracy:  0.984375 \t\t\n",
      "Train Loss:  0.0565905 ; Train accuracy:  0.984375\n",
      "17280/17280 [==============================] - 26s    \n",
      "Test Loss:  0.16334751881 : Test accuracy:  0.971296296296\n",
      "confusions [prediction\\True]\n",
      "[[2791    9    0    9    7    2]\n",
      " [  21 2795    8   58   16    6]\n",
      " [  10   22 2847    7    7    5]\n",
      " [   6   73    3 2798   23   13]\n",
      " [   8   15    2   32 2772   49]\n",
      " [   5   11    7   13   49 2781]]\n",
      "Save best score!! 0.971296296296\n",
      "Elapsed time in epoch 5: 506.14588689804077 [s]\n",
      "Saved model 5!\n",
      "\n",
      "#######\n",
      "epoch6\n",
      "#######\n",
      "Iteration  108  ( 9.91 %) Train Loss:  0.0223792 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  216  ( 19.91 %) Train Loss:  0.0190955 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  324  ( 29.91 %) Train Loss:  0.0215464 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  432  ( 39.91 %) Train Loss:  0.0137875 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  540  ( 49.91 %) Train Loss:  0.0354587 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  648  ( 59.91 %) Train Loss:  0.0809815 ; Train accuracy:  0.984375 \t\t\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration  756  ( 69.91 %) Train Loss:  0.0284208 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  864  ( 79.91 %) Train Loss:  0.033852 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  972  ( 89.91 %) Train Loss:  0.361873 ; Train accuracy:  0.96875 \t\t\n",
      "Iteration  1080  ( 99.91 %) Train Loss:  0.0496482 ; Train accuracy:  1.0 \t\t\n",
      "Train Loss:  0.0496482 ; Train accuracy:  1.0\n",
      "17280/17280 [==============================] - 26s    \n",
      "Test Loss:  0.238211073837 : Test accuracy:  0.96712962963\n",
      "confusions [prediction\\True]\n",
      "[[2748   32   10    6   17    5]\n",
      " [   4 2823   15   31   21   10]\n",
      " [   1   21 2857    2    5   12]\n",
      " [   4  142   11 2690   43   26]\n",
      " [   0   15    2   16 2800   45]\n",
      " [   2   14    6    4   46 2794]]\n",
      "Count early stop!! 1\n",
      "Elapsed time in epoch 6: 506.0443868637085 [s]\n",
      "Saved model 6!\n",
      "\n",
      "#######\n",
      "epoch7\n",
      "#######\n",
      "Iteration  108  ( 9.91 %) Train Loss:  0.0535448 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  216  ( 19.91 %) Train Loss:  0.0761801 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  324  ( 29.91 %) Train Loss:  0.155577 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  432  ( 39.91 %) Train Loss:  0.116288 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  540  ( 49.91 %) Train Loss:  0.116902 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  648  ( 59.91 %) Train Loss:  0.0671258 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  756  ( 69.91 %) Train Loss:  0.0420906 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  864  ( 79.91 %) Train Loss:  0.0568692 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  972  ( 89.91 %) Train Loss:  0.0582834 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  1080  ( 99.91 %) Train Loss:  0.200832 ; Train accuracy:  0.984375 \t\t\n",
      "Train Loss:  0.200832 ; Train accuracy:  0.984375\n",
      "17280/17280 [==============================] - 26s    \n",
      "Test Loss:  0.322649088943 : Test accuracy:  0.967303240741\n",
      "confusions [prediction\\True]\n",
      "[[2770   20    4   16    5    3]\n",
      " [  18 2744   11  106    8   17]\n",
      " [   6   23 2842    7    7   13]\n",
      " [   7   63    4 2794   23   25]\n",
      " [   5   15    5   35 2754   64]\n",
      " [   5   11    6    6   27 2811]]\n",
      "Count early stop!! 2\n",
      "Elapsed time in epoch 7: 503.34569358825684 [s]\n",
      "Saved model 7!\n",
      "\n",
      "#######\n",
      "epoch8\n",
      "#######\n",
      "Iteration  108  ( 9.91 %) Train Loss:  0.0665903 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  216  ( 19.91 %) Train Loss:  0.0526489 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  324  ( 29.91 %) Train Loss:  0.15557 ; Train accuracy:  0.953125 \t\t\n",
      "Iteration  432  ( 39.91 %) Train Loss:  0.0185443 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  540  ( 49.91 %) Train Loss:  0.0319904 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  648  ( 59.91 %) Train Loss:  0.0325097 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  756  ( 69.91 %) Train Loss:  0.0677381 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  864  ( 79.91 %) Train Loss:  0.023115 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  972  ( 89.91 %) Train Loss:  0.0306311 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  1080  ( 99.91 %) Train Loss:  0.0549477 ; Train accuracy:  1.0 \t\t\n",
      "Train Loss:  0.0549477 ; Train accuracy:  1.0\n",
      "17280/17280 [==============================] - 26s    \n",
      "Test Loss:  0.227055878868 : Test accuracy:  0.970023148148\n",
      "confusions [prediction\\True]\n",
      "[[2793    8    4    4    4    5]\n",
      " [  29 2778   12   60   15   10]\n",
      " [   7   20 2857    5    1    8]\n",
      " [   9   77    5 2778   23   24]\n",
      " [  11   13   15   28 2770   41]\n",
      " [   9    6   10    6   49 2786]]\n",
      "Count early stop!! 3\n",
      "Elapsed time in epoch 8: 505.77065348625183 [s]\n",
      "Saved model 8!\n",
      "\n",
      "#######\n",
      "epoch9\n",
      "#######\n",
      "Iteration  108  ( 9.91 %) Train Loss:  0.0460439 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  216  ( 19.91 %) Train Loss:  0.0161478 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  324  ( 29.91 %) Train Loss:  0.0175997 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  432  ( 39.91 %) Train Loss:  0.0131006 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  540  ( 49.91 %) Train Loss:  0.0320871 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  648  ( 59.91 %) Train Loss:  0.0626202 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  756  ( 69.91 %) Train Loss:  0.048917 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  864  ( 79.91 %) Train Loss:  0.0372606 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  972  ( 89.91 %) Train Loss:  0.0738277 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  1080  ( 99.91 %) Train Loss:  0.0551797 ; Train accuracy:  1.0 \t\t\n",
      "Train Loss:  0.0551797 ; Train accuracy:  1.0\n",
      "17280/17280 [==============================] - 26s    \n",
      "Test Loss:  0.21552383701 : Test accuracy:  0.967650462963\n",
      "confusions [prediction\\True]\n",
      "[[2780    6    5   10   14    3]\n",
      " [  30 2681   42  122   17   12]\n",
      " [   4    9 2857   14    3   11]\n",
      " [   3   41   12 2825   24   11]\n",
      " [   2   17    2   33 2782   42]\n",
      " [   7    6    1   16   40 2796]]\n",
      "Count early stop!! 4\n",
      "Elapsed time in epoch 9: 504.45735454559326 [s]\n",
      "Saved model 9!\n",
      "\n",
      "#######\n",
      "epoch10\n",
      "#######\n",
      "Iteration  108  ( 9.91 %) Train Loss:  0.0844658 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  216  ( 19.91 %) Train Loss:  0.0632939 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  324  ( 29.91 %) Train Loss:  0.0191386 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  432  ( 39.91 %) Train Loss:  0.016815 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  540  ( 49.91 %) Train Loss:  0.273767 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  648  ( 59.91 %) Train Loss:  0.0221266 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  756  ( 69.91 %) Train Loss:  0.0325256 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  864  ( 79.91 %) Train Loss:  0.134338 ; Train accuracy:  0.96875 \t\t\n",
      "Iteration  972  ( 89.91 %) Train Loss:  0.189443 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  1080  ( 99.91 %) Train Loss:  0.0408772 ; Train accuracy:  1.0 \t\t\n",
      "Train Loss:  0.0408772 ; Train accuracy:  1.0\n",
      "17280/17280 [==============================] - 26s    \n",
      "Test Loss:  0.22164378496 : Test accuracy:  0.970196759259\n",
      "confusions [prediction\\True]\n",
      "[[2795    4    2    8    5    4]\n",
      " [  37 2745    6   76   20   20]\n",
      " [  11   14 2841   14    5   13]\n",
      " [   7   63    2 2799   21   24]\n",
      " [   8    7    2   25 2759   77]\n",
      " [   6    5    0    6   23 2826]]\n",
      "Count early stop!! 5\n",
      "Elapsed time in epoch 10: 502.0514967441559 [s]\n",
      "Saved model 10!\n",
      "\n",
      "#######\n",
      "epoch11\n",
      "#######\n",
      "Iteration  108  ( 9.91 %) Train Loss:  0.0162251 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  216  ( 19.91 %) Train Loss:  0.0959829 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  324  ( 29.91 %) Train Loss:  0.208444 ; Train accuracy:  0.96875 \t\t\n",
      "Iteration  432  ( 39.91 %) Train Loss:  0.0311375 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  540  ( 49.91 %) Train Loss:  0.0115194 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  648  ( 59.91 %) Train Loss:  0.0147653 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  756  ( 69.91 %) Train Loss:  0.0217462 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  864  ( 79.91 %) Train Loss:  0.064785 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  972  ( 89.91 %) Train Loss:  0.225097 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  1080  ( 99.91 %) Train Loss:  0.0991661 ; Train accuracy:  1.0 \t\t\n",
      "Train Loss:  0.0991661 ; Train accuracy:  1.0\n",
      "17280/17280 [==============================] - 26s    \n",
      "Test Loss:  0.321501562496 : Test accuracy:  0.966898148148\n",
      "confusions [prediction\\True]\n",
      "[[2779   18    3    7    7    4]\n",
      " [  10 2807   17   55    9    6]\n",
      " [   2   19 2860    9    2    6]\n",
      " [   6  102    2 2751   27   28]\n",
      " [   6   57    0   18 2717   80]\n",
      " [   8   19   12   11   22 2794]]\n",
      "Count early stop!! 6\n",
      "Elapsed time in epoch 11: 500.1904282569885 [s]\n",
      "Saved model 11!\n",
      "\n",
      "#######\n",
      "epoch12\n",
      "#######\n",
      "Iteration  108  ( 9.91 %) Train Loss:  0.320841 ; Train accuracy:  0.953125 \t\t\n",
      "Iteration  216  ( 19.91 %) Train Loss:  0.0519162 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  324  ( 29.91 %) Train Loss:  0.086735 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  432  ( 39.91 %) Train Loss:  0.100913 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  540  ( 49.91 %) Train Loss:  0.11356 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  648  ( 59.91 %) Train Loss:  0.0622235 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  756  ( 69.91 %) Train Loss:  0.0429947 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  864  ( 79.91 %) Train Loss:  0.052275 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  972  ( 89.91 %) Train Loss:  0.0874658 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  1080  ( 99.91 %) Train Loss:  0.0363437 ; Train accuracy:  1.0 \t\t\n",
      "Train Loss:  0.0363437 ; Train accuracy:  1.0\n",
      "17280/17280 [==============================] - 26s    \n",
      "Test Loss:  0.191984721094 : Test accuracy:  0.970081018519\n",
      "confusions [prediction\\True]\n",
      "[[2787    8    7    8    7    1]\n",
      " [  21 2784   19   62   13    5]\n",
      " [   6    9 2863   10    4    6]\n",
      " [   6   86    3 2797   16    8]\n",
      " [   5   16    7   44 2767   39]\n",
      " [  13    9    3   25   51 2765]]\n",
      "Count early stop!! 7\n",
      "Elapsed time in epoch 12: 502.91288781166077 [s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved model 12!\n",
      "\n",
      "#######\n",
      "epoch13\n",
      "#######\n",
      "Iteration  108  ( 9.91 %) Train Loss:  0.127907 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  216  ( 19.91 %) Train Loss:  0.213258 ; Train accuracy:  0.96875 \t\t\n",
      "Iteration  324  ( 29.91 %) Train Loss:  0.0295344 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  432  ( 39.91 %) Train Loss:  0.0441871 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  540  ( 49.91 %) Train Loss:  0.0851306 ; Train accuracy:  0.96875 \t\t\n",
      "Iteration  648  ( 59.91 %) Train Loss:  0.0834111 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  756  ( 69.91 %) Train Loss:  0.047237 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  864  ( 79.91 %) Train Loss:  0.0195374 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  972  ( 89.91 %) Train Loss:  0.0563899 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  1080  ( 99.91 %) Train Loss:  0.0400628 ; Train accuracy:  1.0 \t\t\n",
      "Train Loss:  0.0400628 ; Train accuracy:  1.0\n",
      "17280/17280 [==============================] - 26s    \n",
      "Test Loss:  0.254803313398 : Test accuracy:  0.969212962963\n",
      "confusions [prediction\\True]\n",
      "[[2794    6    6    6    5    1]\n",
      " [  24 2794    7   52   20    7]\n",
      " [   6   21 2856    5    8    2]\n",
      " [  10   92    5 2762   27   20]\n",
      " [  13   20    2   30 2782   31]\n",
      " [   9    9    6    7   75 2760]]\n",
      "Count early stop!! 8\n",
      "Elapsed time in epoch 13: 502.071231842041 [s]\n",
      "Saved model 13!\n",
      "\n",
      "#######\n",
      "epoch14\n",
      "#######\n",
      "Iteration  108  ( 9.91 %) Train Loss:  0.216373 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  216  ( 19.91 %) Train Loss:  0.0473716 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  324  ( 29.91 %) Train Loss:  0.022382 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  432  ( 39.91 %) Train Loss:  0.106064 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  540  ( 49.91 %) Train Loss:  0.0499 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  648  ( 59.91 %) Train Loss:  0.0357039 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  756  ( 69.91 %) Train Loss:  0.037715 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  864  ( 79.91 %) Train Loss:  0.161223 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  972  ( 89.91 %) Train Loss:  0.14617 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  1080  ( 99.91 %) Train Loss:  0.101433 ; Train accuracy:  0.984375 \t\t\n",
      "Train Loss:  0.101433 ; Train accuracy:  0.984375\n",
      "17280/17280 [==============================] - 26s    \n",
      "Test Loss:  0.199177005694 : Test accuracy:  0.969965277778\n",
      "confusions [prediction\\True]\n",
      "[[2788    9    5    6    4    6]\n",
      " [  14 2799   19   43   20    9]\n",
      " [   8   14 2858    5    4    9]\n",
      " [  10  110    5 2746   27   18]\n",
      " [   6   14    9   18 2772   59]\n",
      " [   7    6    5    4   46 2798]]\n",
      "Count early stop!! 9\n",
      "Elapsed time in epoch 14: 506.34342885017395 [s]\n",
      "Saved model 14!\n",
      "\n",
      "#######\n",
      "epoch15\n",
      "#######\n",
      "Iteration  108  ( 9.91 %) Train Loss:  0.0293375 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  216  ( 19.91 %) Train Loss:  0.0538783 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  324  ( 29.91 %) Train Loss:  0.0363745 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  432  ( 39.91 %) Train Loss:  0.203672 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  540  ( 49.91 %) Train Loss:  0.0579074 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  648  ( 59.91 %) Train Loss:  0.0862985 ; Train accuracy:  0.984375 \t\t\n",
      "Iteration  756  ( 69.91 %) Train Loss:  0.0607961 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  864  ( 79.91 %) Train Loss:  0.0540465 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  972  ( 89.91 %) Train Loss:  0.0375387 ; Train accuracy:  1.0 \t\t\n",
      "Iteration  1080  ( 99.91 %) Train Loss:  0.102543 ; Train accuracy:  0.984375 \t\t\n",
      "Train Loss:  0.102543 ; Train accuracy:  0.984375\n",
      "17280/17280 [==============================] - 26s    \n",
      "Test Loss:  0.250030092371 : Test accuracy:  0.965277777778\n",
      "confusions [prediction\\True]\n",
      "[[2749   16   27    9    3   14]\n",
      " [  14 2791   24   51   15    9]\n",
      " [   5   14 2859    6    2   12]\n",
      " [  18   93    7 2740   20   38]\n",
      " [  15   18    3   23 2779   40]\n",
      " [  17   12    2    8   65 2762]]\n",
      "Count early stop!! 10\n",
      "Elapsed time in epoch 15: 505.01724910736084 [s]\n",
      "Saved model 15!\n",
      "\n",
      "========\n",
      "Best model\n",
      "========\n",
      "Stop by early stopping\n",
      "Best score:  0.971296296296 Beat model:  4\n",
      "Elapsed time in total: 7570.786213159561\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import time\n",
    "import numpy as np\n",
    "from keras.models import Model, Input\n",
    "from keras.layers import Dense, Dropout, Activation, Embedding, Concatenate, BatchNormalization, regularizers\n",
    "from keras.layers import Conv1D, Flatten, Dropout, MaxPool1D, Lambda, Dot\n",
    "from keras.layers import LSTM, GRU, TimeDistributed, Bidirectional\n",
    "from keras.optimizers import Adam, Adadelta\n",
    "from keras.utils.vis_utils import plot_model\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from keras import backend as K\n",
    "import myLayers.AttentionBase\n",
    "\n",
    "patience = 10\n",
    "num_epoch = 30\n",
    "batch_size = 64\n",
    "save_every = 1\n",
    "pretrain = 0\n",
    "\n",
    "def build_model():\n",
    "    main_input = Input(shape=(max_word_length,), dtype='int32')\n",
    "    \n",
    "    embedder = Embedding(40000+1, 300, input_length=max_word_length)\n",
    "    embed = embedder(main_input)\n",
    "    \n",
    "    rnn = Bidirectional(GRU(150, dropout=0.2, recurrent_dropout=0.1, return_sequences=True, \n",
    "                            kernel_regularizer=regularizers.l2(0.01), \n",
    "                            recurrent_regularizer=regularizers.l2(0.01), \n",
    "                            bias_regularizer=regularizers.l2(0.01)))(embed)\n",
    "    rnn = myLayers.AttentionBase.Attention(150*2, input_metrix=True, \n",
    "                                           kernel_regularizer=regularizers.l2(0.01))(rnn)\n",
    "    rnn = Dense(60, kernel_regularizer=regularizers.l2(0.01), \n",
    "                bias_regularizer=regularizers.l2(0.01))(rnn)\n",
    "    rnn = Dropout(0.5)(rnn)\n",
    "    \n",
    "    main_output = Dense(label_newnum, activation = 'softmax')(rnn)\n",
    "    model = Model(inputs = main_input, outputs = main_output)\n",
    "    #opt = 'adam'\n",
    "    opt = 'Nadam'\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer=opt, metrics = ['accuracy'])\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "if pretrain == 0:\n",
    "    model = build_model()\n",
    "else:\n",
    "    model = load_model(model_name)\n",
    "\n",
    "model_name = 'RNN_PTT_model_fixed_' + str(time.time())\n",
    "mdl_dir = os.path.join(base_dir, 'model')\n",
    "if not os.path.exists(mdl_dir):\n",
    "    os.makedirs(mdl_dir)\n",
    "model_dir = os.path.join(mdl_dir, model_name)\n",
    "if not os.path.exists(model_dir):\n",
    "    os.makedirs(model_dir)\n",
    "\n",
    "num_instances = len(y_train)\n",
    "iter_per_epoch = int(num_instances / batch_size)\n",
    "iter_pct10 = int(iter_per_epoch / 10)\n",
    "if (num_instances % batch_size) > 0:\n",
    "    iter_per_epoch += 1\n",
    "batch_cutoff = [0]\n",
    "for i in range(iter_per_epoch - 1):\n",
    "    batch_cutoff.append(batch_size * (i+1))\n",
    "batch_cutoff.append(num_instances)\n",
    "\n",
    "total_start_t = time.time()\n",
    "best_metrics = 0.0\n",
    "best_epoch = 1\n",
    "early_stop_counter = 0\n",
    "for e in range(num_epoch):\n",
    "    rand_idxs = np.random.permutation(num_instances)\n",
    "    print('\\n#######')\n",
    "    print('epoch' + str(e+1))\n",
    "    print('#######')\n",
    "    start_t = time.time()\n",
    "    for i in range(iter_per_epoch):\n",
    "        X_batch = []\n",
    "        Y_batch = []\n",
    "        # Get random batch input.\n",
    "        for n in range(batch_cutoff[i],batch_cutoff[i+1]):\n",
    "            X_batch.append(x_train[rand_idxs[n]])\n",
    "            Y_batch.append(y_train[rand_idxs[n]])\n",
    "        loss_and_metrics = model.train_on_batch(np.asarray(X_batch),np.asarray(Y_batch))\n",
    "        if i % iter_pct10 == iter_pct10 - 1:\n",
    "            print('Iteration ',i+1,' (',round(i*100/iter_per_epoch,2),'%) Train Loss: ',loss_and_metrics[0],'; Train accuracy: ',loss_and_metrics[1],'\\t\\t',end='\\n')\n",
    "    \n",
    "    print('Train Loss: ',loss_and_metrics[0],'; Train accuracy: ',loss_and_metrics[1])\n",
    "    loss_and_metrics = model.evaluate(x_test, y_test, batch_size)\n",
    "    print('Test Loss: ',loss_and_metrics[0],': Test accuracy: ',loss_and_metrics[1])\n",
    "\n",
    "    predictions = model.predict(x_test)\n",
    "    predictions = predictions.argmax(axis=-1)\n",
    "    y_labels = y_test.argmax(axis=-1)\n",
    "    confusions = confusion_matrix(y_labels, predictions)\n",
    "    print('confusions [prediction\\True]')\n",
    "    print(confusions)\n",
    "\n",
    "    if loss_and_metrics[1] >= best_metrics:\n",
    "        best_metrics = loss_and_metrics[1]\n",
    "        best_epoch = e\n",
    "        early_stop_counter = 0\n",
    "        print('Save best score!! '+str(best_metrics))\n",
    "    else:\n",
    "        early_stop_counter += 1\n",
    "        print('Count early stop!! '+str(early_stop_counter))\n",
    "\n",
    "    print('Elapsed time in epoch ' + str(e+1) + ': ' + str(time.time() - start_t) + ' [s]')\n",
    "\n",
    "    if (e+1) % save_every == 0:\n",
    "        model_path = os.path.join(model_dir, 'model-%d.h5' %(e+1))\n",
    "        model.save(model_path)\n",
    "        model_weights_path = os.path.join(model_dir, 'model_weights-%d.h5' %(e+1))\n",
    "        model.save_weights(model_weights_path)\n",
    "        print('Saved model %s!' %str(e+1))\n",
    "\n",
    "    if patience != 0 and early_stop_counter >= patience:\n",
    "        break\n",
    "\n",
    "print('\\n========')\n",
    "print('Best model')\n",
    "print('========')\n",
    "print('Stop by early stopping')\n",
    "print('Best score: ', best_metrics, 'Beat model: ', best_epoch)\n",
    "plot_model(model, to_file=model_dir+'/model.png')\n",
    "print('Elapsed time in total: ' + str(time.time() - total_start_t))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "default_view": {},
   "name": "multi-PTT text classification model.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}