text-classification/39.fast-slow-lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from utils import *\n",
    "import tensorflow as tf\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['negative', 'positive']\n",
      "10662\n",
      "10662\n"
     ]
    }
   ],
   "source": [
    "trainset = sklearn.datasets.load_files(container_path = 'data', encoding = 'UTF-8')\n",
    "trainset.data, trainset.target = separate_dataset(trainset,1.0)\n",
    "print (trainset.target_names)\n",
    "print (len(trainset.data))\n",
    "print (len(trainset.target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "ONEHOT = np.zeros((len(trainset.data),len(trainset.target_names)))\n",
    "ONEHOT[np.arange(len(trainset.data)),trainset.target] = 1.0\n",
    "train_X, test_X, train_Y, test_Y, train_onehot, test_onehot = train_test_split(trainset.data, \n",
    "                                                                               trainset.target, \n",
    "                                                                               ONEHOT, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 20465\n",
      "Most common words [('the', 10129), ('a', 7312), ('and', 6199), ('of', 6063), ('to', 4233), ('is', 3378)]\n",
      "Sample data [4, 655, 9, 2797, 8, 22, 4, 3527, 12949, 98] ['the', 'rock', 'is', 'destined', 'to', 'be', 'the', '21st', 'centurys', 'new']\n"
     ]
    }
   ],
   "source": [
    "concat = ' '.join(trainset.data).split()\n",
    "vocabulary_size = len(list(set(concat)))\n",
    "data, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size)\n",
    "print('vocab from size: %d'%(vocabulary_size))\n",
    "print('Most common words', count[4:10])\n",
    "print('Sample data', data[:10], [rev_dictionary[i] for i in data[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary['GO']\n",
    "PAD = dictionary['PAD']\n",
    "EOS = dictionary['EOS']\n",
    "UNK = dictionary['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def moments_for_layer_norm(x, axes=1, name=None):\n",
    "    epsilon = 1e-3\n",
    "    if not isinstance(axes, list): axes = [axes]\n",
    "    mean = tf.reduce_mean(x, axes, keep_dims=True)\n",
    "    variance = tf.sqrt(tf.reduce_mean(tf.square(x - mean), axes, keep_dims=True) + epsilon)\n",
    "    return mean, variance\n",
    "\n",
    "def layer_norm_all(h, base, num_units, scope):\n",
    "    with tf.variable_scope(scope):\n",
    "        h_reshape = tf.reshape(h, [-1, base, num_units])\n",
    "        mean = tf.reduce_mean(h_reshape, [2], keep_dims=True)\n",
    "        var = tf.reduce_mean(tf.square(h_reshape - mean), [2], keep_dims=True)\n",
    "        epsilon = tf.constant(1e-3)\n",
    "        rstd = tf.rsqrt(var + epsilon)\n",
    "        h_reshape = (h_reshape - mean) * rstd\n",
    "        h = tf.reshape(h_reshape, [-1, base * num_units])\n",
    "        alpha = tf.get_variable('layer_norm_alpha', [4 * num_units],\n",
    "                                initializer=tf.constant_initializer(1.0), dtype=tf.float32)\n",
    "        bias = tf.get_variable('layer_norm_bias', [4 * num_units],\n",
    "                               initializer=tf.constant_initializer(0.0), dtype=tf.float32)\n",
    "        return (h * alpha) + bias\n",
    "    \n",
    "def layer_norm(x, scope=\"layer_norm\", alpha_start=1.0, bias_start=0.0):\n",
    "    with tf.variable_scope(scope):\n",
    "        num_units = x.get_shape().as_list()[1]\n",
    "        alpha = tf.get_variable('alpha', [num_units],\n",
    "                                initializer=tf.constant_initializer(alpha_start), dtype=tf.float32)\n",
    "        bias = tf.get_variable('bias', [num_units],\n",
    "                               initializer=tf.constant_initializer(bias_start), dtype=tf.float32)\n",
    "        mean, variance = moments_for_layer_norm(x)\n",
    "    return (alpha * (x - mean)) / (variance) + bias\n",
    "\n",
    "def zoneout(new_h, new_c, h, c, h_keep, c_keep, is_training):\n",
    "    mask_c = tf.ones_like(c)\n",
    "    mask_h = tf.ones_like(h)\n",
    "\n",
    "    if is_training:\n",
    "        mask_c = tf.nn.dropout(mask_c, c_keep)\n",
    "        mask_h = tf.nn.dropout(mask_h, h_keep)\n",
    "\n",
    "    mask_c *= c_keep\n",
    "    mask_h *= h_keep\n",
    "\n",
    "    h = new_h * mask_h + (-mask_h + 1.) * h\n",
    "    c = new_c * mask_c + (-mask_c + 1.) * c\n",
    "\n",
    "    return h, c\n",
    "\n",
    "class LN_LSTMCell(tf.contrib.rnn.RNNCell):\n",
    "    def __init__(self, num_units, f_bias=1.0, use_zoneout=False,\n",
    "                 zoneout_keep_h = 0.9, zoneout_keep_c = 0.5, is_training = False):\n",
    "        self.num_units = num_units\n",
    "        self.f_bias = f_bias\n",
    "\n",
    "        self.use_zoneout  = use_zoneout\n",
    "        self.zoneout_keep_h = zoneout_keep_h\n",
    "        self.zoneout_keep_c = zoneout_keep_c\n",
    "\n",
    "        self.is_training = is_training\n",
    "        \n",
    "    def __call__(self, x, state, scope=None):\n",
    "        with tf.variable_scope(scope or type(self).__name__):\n",
    "            h, c = state\n",
    "            h_size = self.num_units\n",
    "            x_size = x.get_shape().as_list()[1]\n",
    "            w_init = tf.constant_initializer(1.0)\n",
    "            h_init = tf.constant_initializer(1.0)\n",
    "            b_init = tf.constant_initializer(0.0)\n",
    "            W_xh = tf.get_variable('W_xh',\n",
    "                                   [x_size, 4 * h_size], initializer=w_init, dtype=tf.float32)\n",
    "            W_hh = tf.get_variable('W_hh',\n",
    "                                   [h_size, 4 * h_size], initializer=h_init, dtype=tf.float32)\n",
    "            bias = tf.get_variable('bias', [4 * h_size], initializer=b_init, dtype=tf.float32)\n",
    "            concat = tf.concat(axis=1, values=[x, h])\n",
    "            W_full = tf.concat(axis=0, values=[W_xh, W_hh])\n",
    "            concat = tf.matmul(concat, W_full) + bias\n",
    "            concat = layer_norm_all(concat, 4, h_size, 'ln')\n",
    "            i, j, f, o = tf.split(axis=1, num_or_size_splits=4, value=concat)\n",
    "            new_c = c * tf.sigmoid(f + self.f_bias) + tf.sigmoid(i) * tf.tanh(j)\n",
    "            new_h = tf.tanh(layer_norm(new_c, 'ln_c')) * tf.sigmoid(o)\n",
    "            if self.use_zoneout:\n",
    "                new_h, new_c = zoneout(new_h, new_c, h, c, self.zoneout_keep_h,\n",
    "                                           self.zoneout_keep_c, self.is_training)\n",
    "        return new_h, (new_h, new_c)\n",
    "\n",
    "    def zero_state(self, batch_size, dtype):\n",
    "        h = tf.zeros([batch_size, self.num_units], dtype=dtype)\n",
    "        c = tf.zeros([batch_size, self.num_units], dtype=dtype)\n",
    "        return (h, c)\n",
    "    \n",
    "\n",
    "class FSRNNCell(tf.contrib.rnn.RNNCell):\n",
    "    def __init__(self, fast_cells, slow_cell, keep_prob=1.0, training=True):\n",
    "        self.fast_layers = len(fast_cells)\n",
    "        assert self.fast_layers >= 2, 'At least two fast layers are needed'\n",
    "        self.fast_cells = fast_cells\n",
    "        self.slow_cell = slow_cell\n",
    "        self.keep_prob = keep_prob\n",
    "        if not training: self.keep_prob = 1.0\n",
    "\n",
    "    def __call__(self, inputs, state, scope='FS-RNN'):\n",
    "        F_state = state[0]\n",
    "        S_state = state[1]\n",
    "\n",
    "        with tf.variable_scope(scope):\n",
    "            inputs = tf.nn.dropout(inputs, self.keep_prob)\n",
    "\n",
    "            with tf.variable_scope('Fast_0'):\n",
    "                F_output, F_state = self.fast_cells[0](inputs, F_state)\n",
    "            F_output_drop = tf.nn.dropout(F_output, self.keep_prob)\n",
    "\n",
    "            with tf.variable_scope('Slow'):\n",
    "                S_output, S_state = self.slow_cell(F_output_drop, S_state)\n",
    "            S_output_drop = tf.nn.dropout(S_output, self.keep_prob)\n",
    "\n",
    "            with tf.variable_scope('Fast_1'):\n",
    "                F_output, F_state = self.fast_cells[1](S_output_drop, F_state)\n",
    "\n",
    "            for i in range(2, self.fast_layers):\n",
    "                with tf.variable_scope('Fast_' + str(i)):\n",
    "                    F_output, F_state = self.fast_cells[i](F_output[:, 0:1] * 0.0, F_state)\n",
    "\n",
    "            F_output_drop = tf.nn.dropout(F_output, self.keep_prob)\n",
    "            return F_output_drop, (F_state, S_state)\n",
    "\n",
    "\n",
    "    def zero_state(self, batch_size, dtype):\n",
    "        F_state = self.fast_cells[0].zero_state(batch_size, dtype)\n",
    "        S_state = self.slow_cell.zero_state(batch_size, dtype)\n",
    "        return (F_state, S_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(self, size_layer, num_layers, fast_layers, embedded_size,\n",
    "                 dict_size, dimension_output, learning_rate, batch_size,\n",
    "                timestamp, is_training=True,zoneout_h = 0.95, zoneout_c = 0.7,\n",
    "                keep_prob=0.75):\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [batch_size, maxlen])\n",
    "        self.Y = tf.placeholder(tf.float32, [batch_size, dimension_output])\n",
    "        encoder_embeddings = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))\n",
    "        encoder_embedded = tf.nn.embedding_lookup(encoder_embeddings, self.X)\n",
    "        \n",
    "        F_cells = [LN_LSTMCell(fast_layers, use_zoneout=True, is_training=is_training,\n",
    "                                      zoneout_keep_h=zoneout_h, zoneout_keep_c=zoneout_c)\n",
    "                   for _ in range(num_layers)]\n",
    "        S_cell  = LN_LSTMCell(size_layer, use_zoneout=True, is_training=is_training,\n",
    "                                     zoneout_keep_h=zoneout_h, zoneout_keep_c=zoneout_c)\n",
    "        FS_cell = FSRNNCell(F_cells, S_cell, keep_prob, is_training)\n",
    "        self._initial_state = FS_cell.zero_state(batch_size, tf.float32)\n",
    "        state = self._initial_state\n",
    "        outputs = []\n",
    "        with tf.variable_scope(\"RNN\"):\n",
    "            for time_step in range(timestamp):\n",
    "                if time_step > 0:\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "                out, state = FS_cell(encoder_embedded[:, time_step, :], state)\n",
    "                outputs.append(out)\n",
    "        outputs = tf.reshape(tf.concat(outputs,axis=1),[batch_size,timestamp,fast_layer])\n",
    "        W = tf.get_variable('w',shape=(fast_layer, dimension_output),initializer=tf.orthogonal_initializer())\n",
    "        b = tf.get_variable('b',shape=(dimension_output),initializer=tf.zeros_initializer())\n",
    "        self.logits = tf.matmul(outputs[:, -1], W) + b\n",
    "        self.cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits, labels = self.Y))\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        correct_pred = tf.equal(tf.argmax(self.logits, 1), tf.argmax(self.Y, 1))\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 64\n",
    "fast_layer = 128\n",
    "num_layers = 2\n",
    "embedded_size = 128\n",
    "dimension_output = len(trainset.target_names)\n",
    "learning_rate = 1e-3\n",
    "maxlen = 50\n",
    "batch_size = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(size_layer,num_layers,fast_layer,\n",
    "              embedded_size,vocabulary_size+4,dimension_output,learning_rate,\n",
    "             batch_size,maxlen)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.000000, current acc: 0.517090\n",
      "time taken: 21.84290075302124\n",
      "epoch: 0, training loss: 0.720922, training acc: 0.514560, valid loss: 0.690760, valid acc: 0.517090\n",
      "\n",
      "epoch: 1, pass acc: 0.517090, current acc: 0.569336\n",
      "time taken: 13.864055633544922\n",
      "epoch: 1, training loss: 0.686381, training acc: 0.551728, valid loss: 0.679708, valid acc: 0.569336\n",
      "\n",
      "epoch: 2, pass acc: 0.569336, current acc: 0.635254\n",
      "time taken: 13.893453598022461\n",
      "epoch: 2, training loss: 0.645373, training acc: 0.625592, valid loss: 0.641775, valid acc: 0.635254\n",
      "\n",
      "epoch: 3, pass acc: 0.635254, current acc: 0.690430\n",
      "time taken: 13.86379337310791\n",
      "epoch: 3, training loss: 0.557973, training acc: 0.720762, valid loss: 0.594735, valid acc: 0.690430\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 5, 0, 0, 0\n",
    "while True:\n",
    "    lasttime = time.time()\n",
    "    if CURRENT_CHECKPOINT == EARLY_STOPPING:\n",
    "        print('break epoch:%d\\n'%(EPOCH))\n",
    "        break\n",
    "        \n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    for i in range(0, (len(train_X) // batch_size) * batch_size, batch_size):\n",
    "        batch_x = str_idx(train_X[i:i+batch_size],dictionary,maxlen)\n",
    "        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                           feed_dict = {model.X : batch_x, model.Y : train_onehot[i:i+batch_size]})\n",
    "        train_loss += loss\n",
    "        train_acc += acc\n",
    "    \n",
    "    for i in range(0, (len(test_X) // batch_size) * batch_size, batch_size):\n",
    "        batch_x = str_idx(test_X[i:i+batch_size],dictionary,maxlen)\n",
    "        acc, loss = sess.run([model.accuracy, model.cost], \n",
    "                           feed_dict = {model.X : batch_x, model.Y : test_onehot[i:i+batch_size]})\n",
    "        test_loss += loss\n",
    "        test_acc += acc\n",
    "    \n",
    "    train_loss /= (len(train_X) // batch_size)\n",
    "    train_acc /= (len(train_X) // batch_size)\n",
    "    test_loss /= (len(test_X) // batch_size)\n",
    "    test_acc /= (len(test_X) // batch_size)\n",
    "    \n",
    "    if test_acc > CURRENT_ACC:\n",
    "        print('epoch: %d, pass acc: %f, current acc: %f'%(EPOCH,CURRENT_ACC, test_acc))\n",
    "        CURRENT_ACC = test_acc\n",
    "        CURRENT_CHECKPOINT = 0\n",
    "    else:\n",
    "        CURRENT_CHECKPOINT += 1\n",
    "        \n",
    "    print('time taken:', time.time()-lasttime)\n",
    "    print('epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'%(EPOCH,train_loss,\n",
    "                                                                                          train_acc,test_loss,\n",
    "                                                                                          test_acc))\n",
    "    EPOCH += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}