text-classification/60.char-sparse.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from utils import *\n",
    "import tensorflow as tf\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import time\n",
    "import random\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['negative', 'positive']\n",
      "10662\n",
      "10662\n"
     ]
    }
   ],
   "source": [
    "trainset = sklearn.datasets.load_files(container_path = 'data', encoding = 'UTF-8')\n",
    "trainset.data, trainset.target = separate_dataset(trainset,1.0)\n",
    "print (trainset.target_names)\n",
    "print (len(trainset.data))\n",
    "print (len(trainset.target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_sparse_matrix_to_sparse_tensor(X, limit = 5):\n",
    "    coo = X.tocoo()\n",
    "    indices = np.mat([coo.row, coo.col]).transpose()\n",
    "    coo.data[coo.data > limit] = limit\n",
    "    return tf.SparseTensorValue(indices, coo.col, coo.shape), tf.SparseTensorValue(indices, coo.data, coo.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "bow_chars = CountVectorizer(ngram_range=(3, 5), analyzer='char_wb', max_features=300000).fit(trainset.data)\n",
    "delattr(bow_chars, 'stop_words_')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "78545"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_shape = bow_chars.transform(trainset.data[:1]).shape[1]\n",
    "feature_shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(self, output_size, vocab_size, learning_rate):\n",
    "        self.X = tf.sparse_placeholder(tf.int32)\n",
    "        self.W = tf.sparse_placeholder(tf.int32)\n",
    "        self.Y = tf.placeholder(tf.int32, [None])\n",
    "        embeddings = tf.Variable(tf.truncated_normal([vocab_size,64]))\n",
    "        embed = tf.nn.embedding_lookup_sparse(embeddings, self.X, self.W, combiner='mean')\n",
    "        self.logits = tf.layers.dense(embed, output_size)\n",
    "        self.cost = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "            logits = self.logits, labels = self.Y))\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        correct_pred = tf.equal(tf.argmax(self.logits, 1,output_type=tf.int32), self.Y)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/gradients_impl.py:112: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "model = Model(len(trainset.target_names), feature_shape, 1e-4)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors = bow_chars.transform(trainset.data)\n",
    "train_X, test_X, train_Y, test_Y = train_test_split(\n",
    "    vectors, trainset.target, test_size = 0.2\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 113.62it/s, accuracy=0.471, cost=0.71] \n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 369.59it/s, accuracy=0.476, cost=0.688]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.61it/s, accuracy=0.5, cost=0.696]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.000000, current acc: 0.517692\n",
      "time taken: 2.533757209777832\n",
      "epoch: 0, training loss: 0.697009, training acc: 0.503935, valid loss: 0.696144, valid acc: 0.517692\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.55it/s, accuracy=0.471, cost=0.699]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 371.24it/s, accuracy=0.524, cost=0.684]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.56it/s, accuracy=0.531, cost=0.691]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, pass acc: 0.517692, current acc: 0.544661\n",
      "time taken: 2.3610126972198486\n",
      "epoch: 1, training loss: 0.691297, training acc: 0.535943, valid loss: 0.691925, valid acc: 0.544661\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.78it/s, accuracy=0.471, cost=0.688]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 383.76it/s, accuracy=0.619, cost=0.68] \n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 123.20it/s, accuracy=0.562, cost=0.686]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2, pass acc: 0.544661, current acc: 0.572813\n",
      "time taken: 2.351262092590332\n",
      "epoch: 2, training loss: 0.685471, training acc: 0.570062, valid loss: 0.687536, valid acc: 0.572813\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.97it/s, accuracy=0.529, cost=0.676]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 383.59it/s, accuracy=0.619, cost=0.676]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.52it/s, accuracy=0.594, cost=0.681]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3, pass acc: 0.572813, current acc: 0.596723\n",
      "time taken: 2.347703456878662\n",
      "epoch: 3, training loss: 0.679279, training acc: 0.602760, valid loss: 0.682802, valid acc: 0.596723\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.23it/s, accuracy=0.588, cost=0.663]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 367.67it/s, accuracy=0.571, cost=0.671]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 123.60it/s, accuracy=0.625, cost=0.675]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 4, pass acc: 0.596723, current acc: 0.620856\n",
      "time taken: 2.350752830505371\n",
      "epoch: 4, training loss: 0.672502, training acc: 0.634520, valid loss: 0.677587, valid acc: 0.620856\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.02it/s, accuracy=0.706, cost=0.649]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 381.26it/s, accuracy=0.619, cost=0.665]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 123.15it/s, accuracy=0.656, cost=0.669]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 5, pass acc: 0.620856, current acc: 0.639854\n",
      "time taken: 2.3480422496795654\n",
      "epoch: 5, training loss: 0.664963, training acc: 0.661694, valid loss: 0.671799, valid acc: 0.639854\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.25it/s, accuracy=0.824, cost=0.633]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 384.90it/s, accuracy=0.667, cost=0.659]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 124.63it/s, accuracy=0.719, cost=0.662]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 6, pass acc: 0.639854, current acc: 0.656978\n",
      "time taken: 2.3421616554260254\n",
      "epoch: 6, training loss: 0.656545, training acc: 0.685585, valid loss: 0.665395, valid acc: 0.656978\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.67it/s, accuracy=0.824, cost=0.617]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 382.75it/s, accuracy=0.714, cost=0.652]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.16it/s, accuracy=0.719, cost=0.654]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 7, pass acc: 0.656978, current acc: 0.672226\n",
      "time taken: 2.3535852432250977\n",
      "epoch: 7, training loss: 0.647194, training acc: 0.706924, valid loss: 0.658383, valid acc: 0.672226\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.32it/s, accuracy=0.824, cost=0.599]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 383.58it/s, accuracy=0.714, cost=0.645]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 124.02it/s, accuracy=0.719, cost=0.645]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 8, pass acc: 0.672226, current acc: 0.681602\n",
      "time taken: 2.3416433334350586\n",
      "epoch: 8, training loss: 0.636926, training acc: 0.723104, valid loss: 0.650820, valid acc: 0.681602\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.90it/s, accuracy=0.882, cost=0.58] \n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 372.86it/s, accuracy=0.667, cost=0.637]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.38it/s, accuracy=0.719, cost=0.635]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 9, pass acc: 0.681602, current acc: 0.688858\n",
      "time taken: 2.3542191982269287\n",
      "epoch: 9, training loss: 0.625816, training acc: 0.737511, valid loss: 0.642800, valid acc: 0.688858\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.86it/s, accuracy=0.882, cost=0.561]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 382.34it/s, accuracy=0.714, cost=0.628]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 123.15it/s, accuracy=0.812, cost=0.624]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 10, pass acc: 0.688858, current acc: 0.698011\n",
      "time taken: 2.350154161453247\n",
      "epoch: 10, training loss: 0.613988, training acc: 0.748181, valid loss: 0.634448, valid acc: 0.698011\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.11it/s, accuracy=0.882, cost=0.542]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 384.17it/s, accuracy=0.714, cost=0.619]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.73it/s, accuracy=0.844, cost=0.613]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 11, pass acc: 0.698011, current acc: 0.704574\n",
      "time taken: 2.3451852798461914\n",
      "epoch: 11, training loss: 0.601603, training acc: 0.757795, valid loss: 0.625898, valid acc: 0.704574\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.77it/s, accuracy=0.882, cost=0.523]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 384.32it/s, accuracy=0.714, cost=0.61] \n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 120.82it/s, accuracy=0.844, cost=0.601]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 12, pass acc: 0.704574, current acc: 0.709731\n",
      "time taken: 2.351118564605713\n",
      "epoch: 12, training loss: 0.588831, training acc: 0.767644, valid loss: 0.617286, valid acc: 0.709731\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.16it/s, accuracy=0.882, cost=0.503]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 382.28it/s, accuracy=0.714, cost=0.6]  \n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 124.04it/s, accuracy=0.844, cost=0.589]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 13, pass acc: 0.709731, current acc: 0.712076\n",
      "time taken: 2.3449511528015137\n",
      "epoch: 13, training loss: 0.575844, training acc: 0.775500, valid loss: 0.608736, valid acc: 0.712076\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.81it/s, accuracy=0.882, cost=0.485]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 385.39it/s, accuracy=0.762, cost=0.591]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 123.14it/s, accuracy=0.844, cost=0.577]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 14, pass acc: 0.712076, current acc: 0.716541\n",
      "time taken: 2.349998712539673\n",
      "epoch: 14, training loss: 0.562796, training acc: 0.784293, valid loss: 0.600353, valid acc: 0.716541\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.76it/s, accuracy=0.882, cost=0.467]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 386.88it/s, accuracy=0.762, cost=0.581]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 121.15it/s, accuracy=0.844, cost=0.565]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 15, pass acc: 0.716541, current acc: 0.721229\n",
      "time taken: 2.3500967025756836\n",
      "epoch: 15, training loss: 0.549820, training acc: 0.791211, valid loss: 0.592222, valid acc: 0.721229\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.83it/s, accuracy=0.882, cost=0.449]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 380.13it/s, accuracy=0.762, cost=0.572]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.92it/s, accuracy=0.844, cost=0.553]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 16, pass acc: 0.721229, current acc: 0.725448\n",
      "time taken: 2.351781129837036\n",
      "epoch: 16, training loss: 0.537023, training acc: 0.795549, valid loss: 0.584405, valid acc: 0.725448\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.88it/s, accuracy=0.882, cost=0.433]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 381.28it/s, accuracy=0.762, cost=0.563]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.78it/s, accuracy=0.844, cost=0.54] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 17, pass acc: 0.725448, current acc: 0.726386\n",
      "time taken: 2.3506393432617188\n",
      "epoch: 17, training loss: 0.524486, training acc: 0.801646, valid loss: 0.576946, valid acc: 0.726386\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.65it/s, accuracy=0.882, cost=0.417]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 367.79it/s, accuracy=0.762, cost=0.554]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.01it/s, accuracy=0.844, cost=0.528]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 18, pass acc: 0.726386, current acc: 0.732949\n",
      "time taken: 2.361037015914917\n",
      "epoch: 18, training loss: 0.512268, training acc: 0.806570, valid loss: 0.569872, valid acc: 0.732949\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 121.77it/s, accuracy=0.882, cost=0.402]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 381.25it/s, accuracy=0.714, cost=0.545]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 121.52it/s, accuracy=0.844, cost=0.517]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 19, pass acc: 0.732949, current acc: 0.735048\n",
      "time taken: 2.3702855110168457\n",
      "epoch: 19, training loss: 0.500407, training acc: 0.810556, valid loss: 0.563194, valid acc: 0.735048\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 121.97it/s, accuracy=0.882, cost=0.387]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 384.39it/s, accuracy=0.714, cost=0.537]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 123.63it/s, accuracy=0.844, cost=0.505]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 20, pass acc: 0.735048, current acc: 0.740674\n",
      "time taken: 2.365511417388916\n",
      "epoch: 20, training loss: 0.488924, training acc: 0.815950, valid loss: 0.556916, valid acc: 0.740674\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.45it/s, accuracy=0.882, cost=0.373]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 372.86it/s, accuracy=0.714, cost=0.528]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.17it/s, accuracy=0.844, cost=0.494]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 21, pass acc: 0.740674, current acc: 0.741611\n",
      "time taken: 2.34474515914917\n",
      "epoch: 21, training loss: 0.477828, training acc: 0.820991, valid loss: 0.551033, valid acc: 0.741611\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.32it/s, accuracy=0.941, cost=0.36] \n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 381.31it/s, accuracy=0.714, cost=0.52] \n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 123.60it/s, accuracy=0.844, cost=0.483]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 22, pass acc: 0.741611, current acc: 0.742080\n",
      "time taken: 2.3425352573394775\n",
      "epoch: 22, training loss: 0.467120, training acc: 0.827309, valid loss: 0.545534, valid acc: 0.742080\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.21it/s, accuracy=0.941, cost=0.348]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 383.46it/s, accuracy=0.714, cost=0.513]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 124.14it/s, accuracy=0.844, cost=0.472]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 23, pass acc: 0.742080, current acc: 0.742549\n",
      "time taken: 2.3434298038482666\n",
      "epoch: 23, training loss: 0.456791, training acc: 0.832116, valid loss: 0.540406, valid acc: 0.742549\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.33it/s, accuracy=0.941, cost=0.336]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 355.71it/s, accuracy=0.714, cost=0.505]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 123.10it/s, accuracy=0.844, cost=0.462]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 24, pass acc: 0.742549, current acc: 0.743018\n",
      "time taken: 2.3550713062286377\n",
      "epoch: 24, training loss: 0.446831, training acc: 0.835633, valid loss: 0.535632, valid acc: 0.743018\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.04it/s, accuracy=0.941, cost=0.325]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 383.81it/s, accuracy=0.714, cost=0.498]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 123.10it/s, accuracy=0.844, cost=0.452]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 25, pass acc: 0.743018, current acc: 0.746768\n",
      "time taken: 2.346632719039917\n",
      "epoch: 25, training loss: 0.437225, training acc: 0.839972, valid loss: 0.531196, valid acc: 0.746768\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.56it/s, accuracy=0.941, cost=0.315]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 385.28it/s, accuracy=0.714, cost=0.492]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 123.15it/s, accuracy=0.844, cost=0.442]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 2.35445499420166\n",
      "epoch: 26, training loss: 0.427957, training acc: 0.844544, valid loss: 0.527079, valid acc: 0.746768\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.08it/s, accuracy=0.941, cost=0.305]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 385.18it/s, accuracy=0.714, cost=0.485]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.47it/s, accuracy=0.844, cost=0.433]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 27, pass acc: 0.746768, current acc: 0.750519\n",
      "time taken: 2.363136053085327\n",
      "epoch: 27, training loss: 0.419010, training acc: 0.847710, valid loss: 0.523266, valid acc: 0.750519\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.73it/s, accuracy=0.941, cost=0.295]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 382.85it/s, accuracy=0.714, cost=0.479]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.64it/s, accuracy=0.844, cost=0.424]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 2.3523943424224854\n",
      "epoch: 28, training loss: 0.410367, training acc: 0.850172, valid loss: 0.519738, valid acc: 0.749581\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.80it/s, accuracy=0.941, cost=0.286]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 380.34it/s, accuracy=0.714, cost=0.473]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.93it/s, accuracy=0.812, cost=0.415]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 29, pass acc: 0.750519, current acc: 0.752394\n",
      "time taken: 2.3522958755493164\n",
      "epoch: 29, training loss: 0.402010, training acc: 0.853807, valid loss: 0.516481, valid acc: 0.752394\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.28it/s, accuracy=0.941, cost=0.277]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 382.93it/s, accuracy=0.714, cost=0.467]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 125.17it/s, accuracy=0.812, cost=0.406]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 30, pass acc: 0.752394, current acc: 0.756614\n",
      "time taken: 2.343083381652832\n",
      "epoch: 30, training loss: 0.393925, training acc: 0.856152, valid loss: 0.513478, valid acc: 0.756614\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.13it/s, accuracy=0.941, cost=0.269]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 385.26it/s, accuracy=0.714, cost=0.462]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 120.81it/s, accuracy=0.812, cost=0.398]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 2.344275712966919\n",
      "epoch: 31, training loss: 0.386095, training acc: 0.859786, valid loss: 0.510717, valid acc: 0.754738\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.38it/s, accuracy=0.941, cost=0.261]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 383.59it/s, accuracy=0.714, cost=0.457]\n",
      "train minibatch loop:   4%|▍         | 12/267 [00:00<00:02, 119.69it/s, accuracy=0.812, cost=0.39] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 2.358192205429077\n",
      "epoch: 32, training loss: 0.378507, training acc: 0.863655, valid loss: 0.508183, valid acc: 0.755676\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.64it/s, accuracy=0.941, cost=0.254]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 381.42it/s, accuracy=0.714, cost=0.452]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 123.18it/s, accuracy=0.812, cost=0.383]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 33, pass acc: 0.756614, current acc: 0.757083\n",
      "time taken: 2.3544912338256836\n",
      "epoch: 33, training loss: 0.371146, training acc: 0.867994, valid loss: 0.505864, valid acc: 0.757083\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.85it/s, accuracy=0.941, cost=0.247]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 382.11it/s, accuracy=0.714, cost=0.447]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.02it/s, accuracy=0.812, cost=0.375]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 34, pass acc: 0.757083, current acc: 0.757551\n",
      "time taken: 2.350299835205078\n",
      "epoch: 34, training loss: 0.364000, training acc: 0.870573, valid loss: 0.503750, valid acc: 0.757551\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.67it/s, accuracy=0.941, cost=0.24] \n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 367.55it/s, accuracy=0.714, cost=0.443]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.39it/s, accuracy=0.812, cost=0.368]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 35, pass acc: 0.757551, current acc: 0.758489\n",
      "time taken: 2.361063003540039\n",
      "epoch: 35, training loss: 0.357058, training acc: 0.873856, valid loss: 0.501829, valid acc: 0.758489\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.01it/s, accuracy=0.941, cost=0.233]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 382.51it/s, accuracy=0.714, cost=0.439]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 123.04it/s, accuracy=0.812, cost=0.361]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 36, pass acc: 0.758489, current acc: 0.760364\n",
      "time taken: 2.347468852996826\n",
      "epoch: 36, training loss: 0.350308, training acc: 0.875966, valid loss: 0.500091, valid acc: 0.760364\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.88it/s, accuracy=0.941, cost=0.227]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 382.25it/s, accuracy=0.762, cost=0.435]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 121.92it/s, accuracy=0.844, cost=0.354]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 37, pass acc: 0.760364, current acc: 0.761548\n",
      "time taken: 2.3502254486083984\n",
      "epoch: 37, training loss: 0.343741, training acc: 0.878546, valid loss: 0.498528, valid acc: 0.761548\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.10it/s, accuracy=0.941, cost=0.221]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 366.58it/s, accuracy=0.762, cost=0.431]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 121.57it/s, accuracy=0.844, cost=0.348]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 38, pass acc: 0.761548, current acc: 0.762954\n",
      "time taken: 2.353976249694824\n",
      "epoch: 38, training loss: 0.337347, training acc: 0.881946, valid loss: 0.497132, valid acc: 0.762954\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.49it/s, accuracy=0.941, cost=0.216]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 384.86it/s, accuracy=0.762, cost=0.428]\n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.44it/s, accuracy=0.844, cost=0.342]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 2.3555920124053955\n",
      "epoch: 39, training loss: 0.331117, training acc: 0.883822, valid loss: 0.495894, valid acc: 0.762485\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.08it/s, accuracy=0.941, cost=0.21] \n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 384.55it/s, accuracy=0.81, cost=0.424] \n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 124.30it/s, accuracy=0.875, cost=0.336]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 40, pass acc: 0.762954, current acc: 0.764606\n",
      "time taken: 2.3455002307891846\n",
      "epoch: 40, training loss: 0.325044, training acc: 0.887457, valid loss: 0.494808, valid acc: 0.764606\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.06it/s, accuracy=0.941, cost=0.205]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 380.67it/s, accuracy=0.81, cost=0.421] \n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.85it/s, accuracy=0.875, cost=0.33] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 41, pass acc: 0.764606, current acc: 0.765075\n",
      "time taken: 2.347612142562866\n",
      "epoch: 41, training loss: 0.319119, training acc: 0.889684, valid loss: 0.493867, valid acc: 0.765075\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.50it/s, accuracy=0.941, cost=0.2]  \n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 382.20it/s, accuracy=0.81, cost=0.418] \n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.48it/s, accuracy=0.875, cost=0.324]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 2.3566336631774902\n",
      "epoch: 42, training loss: 0.313337, training acc: 0.891678, valid loss: 0.493064, valid acc: 0.764137\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.63it/s, accuracy=0.941, cost=0.195]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 381.35it/s, accuracy=0.81, cost=0.416] \n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 123.52it/s, accuracy=0.875, cost=0.319]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 43, pass acc: 0.765075, current acc: 0.766013\n",
      "time taken: 2.355032444000244\n",
      "epoch: 43, training loss: 0.307690, training acc: 0.892381, valid loss: 0.492394, valid acc: 0.766013\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.07it/s, accuracy=0.941, cost=0.19] \n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 383.96it/s, accuracy=0.81, cost=0.413] \n",
      "train minibatch loop:   4%|▍         | 12/267 [00:00<00:02, 117.09it/s, accuracy=0.781, cost=0.366]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 44, pass acc: 0.766013, current acc: 0.766481\n",
      "time taken: 2.346391201019287\n",
      "epoch: 44, training loss: 0.302173, training acc: 0.894609, valid loss: 0.491852, valid acc: 0.766481\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 121.30it/s, accuracy=0.941, cost=0.186]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 282.09it/s, accuracy=0.81, cost=0.411] \n",
      "train minibatch loop:   4%|▍         | 12/267 [00:00<00:02, 116.97it/s, accuracy=0.812, cost=0.361]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 2.440406322479248\n",
      "epoch: 45, training loss: 0.296780, training acc: 0.896367, valid loss: 0.491432, valid acc: 0.766013\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.90it/s, accuracy=0.941, cost=0.182]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 382.41it/s, accuracy=0.81, cost=0.409] \n",
      "train minibatch loop:   4%|▍         | 12/267 [00:00<00:02, 117.44it/s, accuracy=0.875, cost=0.303]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 46, pass acc: 0.766481, current acc: 0.766950\n",
      "time taken: 2.349499464035034\n",
      "epoch: 46, training loss: 0.291506, training acc: 0.899064, valid loss: 0.491130, valid acc: 0.766950\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.70it/s, accuracy=0.941, cost=0.177]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 382.25it/s, accuracy=0.81, cost=0.407] \n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 122.72it/s, accuracy=0.875, cost=0.299]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 47, pass acc: 0.766950, current acc: 0.767888\n",
      "time taken: 2.3532044887542725\n",
      "epoch: 47, training loss: 0.286346, training acc: 0.900940, valid loss: 0.490942, valid acc: 0.767888\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.11it/s, accuracy=0.941, cost=0.173]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 384.75it/s, accuracy=0.81, cost=0.405] \n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 123.03it/s, accuracy=0.875, cost=0.294]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 48, pass acc: 0.767888, current acc: 0.769763\n",
      "time taken: 2.3447301387786865\n",
      "epoch: 48, training loss: 0.281295, training acc: 0.903637, valid loss: 0.490863, valid acc: 0.769763\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 123.29it/s, accuracy=0.941, cost=0.169]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 366.54it/s, accuracy=0.81, cost=0.403] \n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 123.67it/s, accuracy=0.875, cost=0.289]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 2.350616455078125\n",
      "epoch: 49, training loss: 0.276349, training acc: 0.905395, valid loss: 0.490889, valid acc: 0.768825\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.84it/s, accuracy=0.941, cost=0.166]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 382.29it/s, accuracy=0.81, cost=0.402] \n",
      "train minibatch loop:   5%|▍         | 13/267 [00:00<00:02, 121.66it/s, accuracy=0.875, cost=0.285]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 2.3508963584899902\n",
      "epoch: 50, training loss: 0.271505, training acc: 0.908327, valid loss: 0.491018, valid acc: 0.767419\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 267/267 [00:02<00:00, 122.67it/s, accuracy=0.941, cost=0.162]\n",
      "test minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 382.51it/s, accuracy=0.81, cost=0.4]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 2.3538389205932617\n",
      "epoch: 51, training loss: 0.266758, training acc: 0.911023, valid loss: 0.491244, valid acc: 0.766950\n",
      "\n",
      "break epoch:52\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import time\n",
    "\n",
    "batch_size = 32\n",
    "EARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 3, 0, 0, 0\n",
    "\n",
    "while True:\n",
    "    lasttime = time.time()\n",
    "    if CURRENT_CHECKPOINT == EARLY_STOPPING:\n",
    "        print('break epoch:%d\\n' % (EPOCH))\n",
    "        break\n",
    "\n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    pbar = tqdm(\n",
    "        range(0, train_X.shape[0], batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        batch_x = convert_sparse_matrix_to_sparse_tensor(train_X[i : min(i + batch_size, train_X.shape[0])])\n",
    "        batch_y = train_Y[i : min(i + batch_size, train_X.shape[0])]\n",
    "        batch_x_expand = np.expand_dims(batch_x,axis = 1)\n",
    "        acc, cost, _ = sess.run(\n",
    "            [model.accuracy, model.cost, model.optimizer],\n",
    "            feed_dict = {\n",
    "                model.Y: batch_y,\n",
    "                model.X: batch_x[0],\n",
    "                model.W: batch_x[1],\n",
    "            },\n",
    "        )\n",
    "        assert not np.isnan(cost)\n",
    "        train_loss += cost\n",
    "        train_acc += acc\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "\n",
    "    pbar = tqdm(range(0, test_X.shape[0], batch_size), desc = 'test minibatch loop')\n",
    "    for i in pbar:\n",
    "        batch_x = convert_sparse_matrix_to_sparse_tensor(test_X[i : min(i + batch_size, test_X.shape[0])])\n",
    "        batch_y = test_Y[i : min(i + batch_size, test_X.shape[0])]\n",
    "        batch_x_expand = np.expand_dims(batch_x,axis = 1)\n",
    "        acc, cost = sess.run(\n",
    "            [model.accuracy, model.cost],\n",
    "            feed_dict = {\n",
    "                model.Y: batch_y,\n",
    "                model.X: batch_x[0],\n",
    "                model.W: batch_x[1],\n",
    "            },\n",
    "        )\n",
    "        test_loss += cost\n",
    "        test_acc += acc\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "\n",
    "    train_loss /= train_X.shape[0] / batch_size\n",
    "    train_acc /= train_X.shape[0] / batch_size\n",
    "    test_loss /= test_X.shape[0] / batch_size\n",
    "    test_acc /= test_X.shape[0] / batch_size\n",
    "\n",
    "    if test_acc > CURRENT_ACC:\n",
    "        print(\n",
    "            'epoch: %d, pass acc: %f, current acc: %f'\n",
    "            % (EPOCH, CURRENT_ACC, test_acc)\n",
    "        )\n",
    "        CURRENT_ACC = test_acc\n",
    "        CURRENT_CHECKPOINT = 0\n",
    "    else:\n",
    "        CURRENT_CHECKPOINT += 1\n",
    "\n",
    "    print('time taken:', time.time() - lasttime)\n",
    "    print(\n",
    "        'epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'\n",
    "        % (EPOCH, train_loss, train_acc, test_loss, test_acc)\n",
    "    )\n",
    "    EPOCH += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "validation minibatch loop: 100%|██████████| 67/67 [00:00<00:00, 447.62it/s]\n"
     ]
    }
   ],
   "source": [
    "real_Y, predict_Y = [], []\n",
    "\n",
    "pbar = tqdm(\n",
    "    range(0, test_X.shape[0], batch_size), desc = 'validation minibatch loop'\n",
    ")\n",
    "for i in pbar:\n",
    "    batch_x = convert_sparse_matrix_to_sparse_tensor(test_X[i : min(i + batch_size, test_X.shape[0])])\n",
    "    batch_y = test_Y[i : min(i + batch_size, test_X.shape[0])]\n",
    "    predict_Y += np.argmax(\n",
    "        sess.run(\n",
    "            model.logits, feed_dict = {model.X: batch_x[0], model.W: batch_x[1], model.Y: batch_y}\n",
    "        ),\n",
    "        1,\n",
    "    ).tolist()\n",
    "    real_Y += batch_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.76      0.76      0.76      1061\n",
      "   positive       0.76      0.77      0.76      1072\n",
      "\n",
      "avg / total       0.76      0.76      0.76      2133\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "print(metrics.classification_report(real_Y, predict_Y, target_names = ['negative','positive']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}