SRNN tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# [Reference](https://mp.weixin.qq.com/s/veyEcUUqsJZ5bAy96o1x2g)\n",
    "# [論文](https://arxiv.org/pdf/1807.02291.pdf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Architecture\n",
    "\n",
    "![architecture](https://mmbiz.qpic.cn/mmbiz_png/UicQ7HgWiaUb3Mic9aQLPQufHfq3zhXgdRDdrjCoA68XTrOOPU8uEd6kB3EicbZFfKKe1qTAzw81vRYeQibwdaSBZTQ/640?wx_fmt=png&tp=webp&wxfrom=5&wx_lazy=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set hyperparameters "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.preprocessing.text import Tokenizer, text_to_word_sequence\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Model\n",
    "from keras.layers import Input, Embedding, GRU, TimeDistributed, Dense\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set hyper parameters\n",
    "MAX_NUM_WORDS = 30000\n",
    "EMBEDDING_DIM = 200\n",
    "VALIDATION_SPLIT = 0.1\n",
    "TEST_SPLIT=0.1\n",
    "NUM_FILTERS = 50\n",
    "MAX_LEN = 512\n",
    "Batch_size = 100\n",
    "EPOCHS = 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(MAX_NUM_WORDS + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            input_length=8,\n",
    "                            trainable=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.optimizers import Adam\n",
    "opt = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 定義layer0\n",
    "\n",
    "最底層的layer\n",
    "\n",
    "他的input是小塊小塊已經切割過的subsequence\n",
    "\n",
    "然後output一個dimension 50的vector來表示這段subsequence\n",
    "\n",
    "相當於讀八個字然後回傳句子的vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_17 (InputLayer)        (None, 8)                 0         \n",
      "_________________________________________________________________\n",
      "embedding_2 (Embedding)      (None, 8, 200)            6000200   \n",
      "_________________________________________________________________\n",
      "gru_12 (GRU)                 (None, 50)                37650     \n",
      "=================================================================\n",
      "Total params: 6,037,850\n",
      "Trainable params: 6,037,850\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "[[1 2 3 4 5 6 7 8]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/zhangtaiwei/htdocs/learning/venv/lib/python3.6/site-packages/ipykernel_launcher.py:15: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "1/1 [==============================] - 1s 1s/step - loss: 808.5721\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[-4.33045402e-02,  2.22507976e-02, -7.09787291e-03,\n",
       "         2.38983631e-02, -1.33954752e-02, -3.16630900e-02,\n",
       "        -1.69239305e-02, -1.04079498e-02, -1.77585874e-02,\n",
       "         1.87187530e-02,  3.52477655e-04,  4.90479497e-03,\n",
       "        -1.63943553e-03,  7.01367948e-03, -1.75614376e-03,\n",
       "        -3.73084247e-02, -1.44285643e-02, -1.10854837e-03,\n",
       "         9.46864206e-03, -1.57715078e-03, -1.34417685e-02,\n",
       "         7.76076363e-03,  1.85797196e-02,  4.44182660e-03,\n",
       "         5.67053817e-03, -3.32920253e-02,  1.93182975e-02,\n",
       "        -8.60072393e-03,  1.31879058e-02,  1.90459210e-02,\n",
       "        -1.58739276e-05, -2.74105780e-02, -1.14374813e-02,\n",
       "        -1.46071501e-02, -1.65605955e-02,  4.81198356e-02,\n",
       "         6.04520552e-03, -1.01083210e-02, -7.87756406e-03,\n",
       "         3.99874821e-02, -2.05743182e-02,  1.06296353e-02,\n",
       "         1.25798304e-03,  7.34724221e-04, -1.00723095e-02,\n",
       "         2.25524567e-02, -1.50563419e-02,  3.61186406e-03,\n",
       "         9.29534156e-03,  2.75120176e-02]], dtype=float32)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input1 = Input(shape=(8,), dtype='int32')\n",
    "embed = embedding_layer(input1)\n",
    "gru1 = GRU(NUM_FILTERS,recurrent_activation='sigmoid',activation=None,return_sequences=False)(embed)\n",
    "Encoder1 = Model(input1, gru1)\n",
    "print(Encoder1.summary())\n",
    "\n",
    "x_train_padded_seqs_split = np.arange(1, 9).reshape(1, 8)\n",
    "print(x_train_padded_seqs_split)\n",
    "y_train = np.arange(50).reshape(1, 50)\n",
    "Encoder1.compile(loss='mse',\n",
    "              optimizer=opt)\n",
    "Encoder1.fit(np.array(x_train_padded_seqs_split), y_train, \n",
    "          nb_epoch = EPOCHS, \n",
    "          batch_size = Batch_size,\n",
    "          verbose = 1)\n",
    "Encoder1.predict(x_train_padded_seqs_split)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TimeDistributed\n",
    "\n",
    "是一個可以平行運算的Wrapper\n",
    "\n",
    "可以平行!!!\n",
    "\n",
    "可以平行!!!\n",
    "\n",
    "可以平行!!!\n",
    "\n",
    "所以下方的layer1\n",
    "\n",
    "送入一個 8*8的 input時長成這樣\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[ 0  1  2  3  4  5  6  7]\n",
      "  [ 8  9 10 11 12 13 14 15]\n",
      "  [16 17 18 19 20 21 22 23]\n",
      "  [24 25 26 27 28 29 30 31]\n",
      "  [32 33 34 35 36 37 38 39]\n",
      "  [40 41 42 43 44 45 46 47]\n",
      "  [48 49 50 51 52 53 54 55]\n",
      "  [56 57 58 59 60 61 62 63]]]\n"
     ]
    }
   ],
   "source": [
    "x_train_padded_seqs_split = np.arange(64).reshape(1, 8, 8)\n",
    "print(x_train_padded_seqs_split)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "會被 `TimeDistributed(Encoder1)(input2)` 進行轉換\n",
    "\n",
    "input2的shape是8*8\n",
    "\n",
    "Encoder是吃(8, )的input, return (50, )\n",
    "\n",
    "所以TimeDistributed會把 (8 X 8) 轉換成 (8 X 50)\n",
    "\n",
    "而且這一步是平行運算\n",
    "\n",
    "並且共用同一個Encoder1的參數\n",
    "\n",
    "實際看看結果吧："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/zhangtaiwei/htdocs/learning/venv/lib/python3.6/site-packages/ipykernel_launcher.py:11: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  # This is added back by InteractiveShellApp.init_path()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "1/1 [==============================] - 1s 888ms/step - loss: 53131.6328\n",
      "(1, 8, 50)\n"
     ]
    }
   ],
   "source": [
    "input2 = Input(shape=(8,8,), dtype='int32')\n",
    "embed2 = TimeDistributed(Encoder1)(input2)\n",
    "Encoder2 = Model(input2, embed2)\n",
    "\n",
    "x_train_padded_seqs_split = np.arange(64).reshape(1, 8, 8)\n",
    "y_train = np.arange(400).reshape(1, 8, 50)\n",
    "Encoder2.compile(loss='mse',\n",
    "              optimizer=opt)\n",
    "Encoder2.fit(np.array(x_train_padded_seqs_split), y_train, \n",
    "          nb_epoch = EPOCHS, \n",
    "          batch_size = Batch_size,\n",
    "          verbose = 1)\n",
    "result = Encoder2.predict(x_train_padded_seqs_split)\n",
    "print(result.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 定義layer1\n",
    "\n",
    "第二層的layer\n",
    "\n",
    "他的input是subsequence的集合\n",
    "\n",
    "layer0的input長度為8\n",
    "\n",
    "layer1的input就是layer0的集合 -> 8*8 ，也就是8個layer0的input sequence的集合\n",
    "\n",
    "在TimeDistributed裡面呼叫Encoder1就會把長度為8的sequence送入layer0\n",
    "\n",
    "會回傳8個句子的vector -> shape:(8*50)\n",
    "\n",
    "而layer1消化過這8個句子的vector\n",
    "\n",
    "回傳一個段落的vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/zhangtaiwei/htdocs/learning/venv/lib/python3.6/site-packages/ipykernel_launcher.py:13: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  del sys.path[0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "1/1 [==============================] - 1s 1s/step - loss: 808.5722\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_16 (InputLayer)        (None, 8, 8)              0         \n",
      "_________________________________________________________________\n",
      "time_distributed_12 (TimeDis (None, 8, 50)             6037850   \n",
      "_________________________________________________________________\n",
      "gru_11 (GRU)                 (None, 50)                15150     \n",
      "=================================================================\n",
      "Total params: 6,053,000\n",
      "Trainable params: 6,053,000\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "(1, 50)\n",
      "[[ 0.00276607  0.0097453  -0.01522984  0.00735891  0.00378472 -0.00459259\n",
      "   0.00639305  0.00152924  0.02014573  0.00158749  0.00433686 -0.01937744\n",
      "  -0.00466379 -0.00158728  0.00098629  0.01804274 -0.01492085  0.01148123\n",
      "   0.00341437 -0.00224343 -0.00989759 -0.00469353  0.01509546 -0.00596046\n",
      "   0.00934241  0.00770333  0.01512451 -0.00593824  0.00352744  0.00927674\n",
      "  -0.01748044 -0.01139651 -0.00336027  0.01935867  0.00939001 -0.00464996\n",
      "   0.01473329 -0.00115261  0.009728    0.0129708   0.00559154  0.0038898\n",
      "  -0.00742476 -0.00466956  0.00977357 -0.02216711  0.00610822  0.01525378\n",
      "  -0.01184907 -0.01060573]]\n"
     ]
    }
   ],
   "source": [
    "input2 = Input(shape=(8,8,), dtype='int32')\n",
    "embed2 = TimeDistributed(Encoder1)(input2)\n",
    "gru2 = GRU(50,recurrent_activation='sigmoid',activation=None,return_sequences=False)(embed2)\n",
    "Encoder2 = Model(input2,gru2)\n",
    "\n",
    "x_train_padded_seqs_split = np.arange(64).reshape(1, 8, 8)\n",
    "y_train = np.arange(50).reshape(1, 50)\n",
    "Encoder2.compile(loss='mse',\n",
    "              optimizer=opt)\n",
    "Encoder2.fit(np.array(x_train_padded_seqs_split), y_train, \n",
    "          nb_epoch = EPOCHS, \n",
    "          batch_size = Batch_size,\n",
    "          verbose = 1)\n",
    "result = Encoder2.predict(x_train_padded_seqs_split)\n",
    "print(Encoder2.summary())\n",
    "print(result.shape)\n",
    "print(result)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}