03.02.02.text_classification_Conv1D_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Classification with 1-d CNN Encoding and LSTM\n",
    "\n",
    "이전 실습들을 통해 LSTM 등의 RNN을 이용하여 Text를 state vector로 encoding하거나, 혹은 1-d CNN을 이용하여 encoding하여, 이를 Feature 로 활용하여 Text classification하는 IMDB dataset 기반 sentimental analysis task를 다루어 보았습니다.\n",
    "\n",
    "이번 실습처럼 500 이상의 시퀀스 길이를 가지는 경우 RNN의 학습 속도가 너무 느리다는 문제가 있습니다. CNN을 사용하면 GPU의 병렬처리 연산 효율을 극대화하여 학습 속도가 매우 빠르면서도 정확도가 꽤 높은 결과를 확인할 수 있었습니다. 그러나, 수백개의 단어 및 10개 이상의 문장으로 이루어진 긴 글을 분석함에 있어서 CNN만 사용했을 때 state 변화 개념이 들어가지 않은 채 특정 키워드의 분포만으로 결론을 단정짓게 되는 구조적인 약점 또한 존재합니다.  \n",
    "\n",
    "만약 그렇다면, CNN과 RNN의 장점을 결합한 모델을 생각해 볼 수는 없을까요? 긴 길이의 입력 시퀀스를 먼저 1-d CNN을 사용하여 보다 짧은 길이의 시퀀스로 인코딩하여 변환한 후 다시 RNN을 적용하는 방식으로 동일한 task를 다시 다루어 보도록 하겠습니다.\n",
    "\n",
    "(참고)  \n",
    "https://github.com/gilbutITbook/006975/blob/master/6.4-sequence-processing-with-convnets.ipynb  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from scipy.spatial.distance import cdist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tf.keras.models import Sequential  # This does not work!\n",
    "from tensorflow.python.keras.models import Sequential\n",
    "from tensorflow.python.keras.layers import Dense, Conv1D, MaxPooling1D, GlobalMaxPooling1D, GRU, Embedding\n",
    "from tensorflow.python.keras.optimizers import Adam\n",
    "from tensorflow.python.keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train-set size:  25000\n",
      "Test-set size:   25000\n"
     ]
    }
   ],
   "source": [
    "# import imdb\n",
    "from keras.datasets import imdb\n",
    "\n",
    "# 데이터 관련 설정은 LSTM 케이스와 동일하게 한다.\n",
    "max_features = 10000\n",
    "max_tokens = 580\n",
    "embedding_size = 8\n",
    "\n",
    "# save np.load\n",
    "np_load_old = np.load\n",
    "# modify the default parameters of np.load\n",
    "np.load = lambda *a,**k: np_load_old(*a, allow_pickle=True, **k)\n",
    "# call load_data with allow_pickle implicitly set to true\n",
    "(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)   # 원래는 이 라인만 있으면 된다.\n",
    "# restore np.load for future normal usage\n",
    "np.load = np_load_old\n",
    "\n",
    "print(\"Train-set size: \", len(x_train))\n",
    "print(\"Test-set size:  \", len(x_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25000, 580)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pad = 'pre'\n",
    "x_train_pad = pad_sequences(x_train, maxlen=max_tokens, padding=pad, truncating=pad)\n",
    "x_test_pad = pad_sequences(x_test, maxlen=max_tokens, padding=pad, truncating=pad)\n",
    "x_train_pad.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Model with Conv1D and GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "layer_embedding (Embedding)  (None, 580, 8)            80000     \n",
      "_________________________________________________________________\n",
      "conv1d_5 (Conv1D)            (None, 574, 32)           1824      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_4 (MaxPooling1 (None, 114, 32)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_6 (Conv1D)            (None, 108, 32)           7200      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_5 (MaxPooling1 (None, 21, 32)            0         \n",
      "_________________________________________________________________\n",
      "gru (GRU)                    (None, 32)                6240      \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 33        \n",
      "=================================================================\n",
      "Total params: 95,297\n",
      "Trainable params: 95,297\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=max_features,\n",
    "                    output_dim=embedding_size,\n",
    "                    input_length=max_tokens,\n",
    "                    name='layer_embedding'))\n",
    "model.add(Conv1D(32, 7, activation='relu'))\n",
    "model.add(MaxPooling1D(5))\n",
    "model.add(Conv1D(32, 7, activation='relu'))\n",
    "model.add(MaxPooling1D(5))\n",
    "\n",
    "#########################################\n",
    "# model.add(GlobalMaxPooling1D())\n",
    "model.add(GRU(units=32))    # 03.02.01과 이 한줄만 다름\n",
    "#########################################\n",
    "\n",
    "model.add(Dense(1))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "model.compile(optimizer=Adam(lr=1e-5),\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'layer_embedding_2/embedding_lookup/Identity_1:0' shape=(?, 580, 8) dtype=float32>,\n",
       " <tf.Tensor 'conv1d_5/Relu:0' shape=(?, 574, 32) dtype=float32>,\n",
       " <tf.Tensor 'max_pooling1d_4/Squeeze:0' shape=(?, 114, 32) dtype=float32>,\n",
       " <tf.Tensor 'conv1d_6/Relu:0' shape=(?, 108, 32) dtype=float32>,\n",
       " <tf.Tensor 'max_pooling1d_5/Squeeze:0' shape=(?, 21, 32) dtype=float32>,\n",
       " <tf.Tensor 'gru/strided_slice_12:0' shape=(?, 32) dtype=float32>,\n",
       " <tf.Tensor 'dense_2/BiasAdd:0' shape=(?, 1) dtype=float32>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_outputs = [layer.output for layer in model.layers]\n",
    "layer_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 23750 samples, validate on 1250 samples\n",
      "WARNING:tensorflow:From C:\\Users\\uhmpp\\Anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Epoch 1/100\n",
      "23750/23750 [==============================] - 19s 784us/sample - loss: 1.7284 - acc: 0.5000 - val_loss: 1.4043 - val_acc: 0.5008\n",
      "Epoch 2/100\n",
      "23750/23750 [==============================] - 16s 694us/sample - loss: 1.2608 - acc: 0.5000 - val_loss: 1.1291 - val_acc: 0.5008\n",
      "Epoch 3/100\n",
      "23750/23750 [==============================] - 17s 703us/sample - loss: 1.0329 - acc: 0.5000 - val_loss: 0.9454 - val_acc: 0.5008\n",
      "Epoch 4/100\n",
      "23750/23750 [==============================] - 17s 713us/sample - loss: 0.8792 - acc: 0.5000 - val_loss: 0.8178 - val_acc: 0.5008\n",
      "Epoch 5/100\n",
      "23750/23750 [==============================] - 17s 704us/sample - loss: 0.7766 - acc: 0.5000 - val_loss: 0.7418 - val_acc: 0.5008\n",
      "Epoch 6/100\n",
      "23750/23750 [==============================] - 18s 748us/sample - loss: 0.7219 - acc: 0.5000 - val_loss: 0.7074 - val_acc: 0.5008\n",
      "Epoch 7/100\n",
      "23750/23750 [==============================] - 19s 791us/sample - loss: 0.7001 - acc: 0.5040 - val_loss: 0.6966 - val_acc: 0.5240\n",
      "Epoch 8/100\n",
      "23750/23750 [==============================] - 18s 747us/sample - loss: 0.6940 - acc: 0.5292 - val_loss: 0.6939 - val_acc: 0.5368\n",
      "Epoch 9/100\n",
      "23750/23750 [==============================] - 17s 731us/sample - loss: 0.6919 - acc: 0.5463 - val_loss: 0.6924 - val_acc: 0.5312\n",
      "Epoch 10/100\n",
      "23750/23750 [==============================] - 17s 722us/sample - loss: 0.6903 - acc: 0.5539 - val_loss: 0.6908 - val_acc: 0.5352\n",
      "Epoch 11/100\n",
      "23750/23750 [==============================] - 17s 708us/sample - loss: 0.6886 - acc: 0.5694 - val_loss: 0.6891 - val_acc: 0.5472\n",
      "Epoch 12/100\n",
      "23750/23750 [==============================] - 17s 731us/sample - loss: 0.6868 - acc: 0.5845 - val_loss: 0.6873 - val_acc: 0.5728\n",
      "Epoch 13/100\n",
      "23750/23750 [==============================] - 19s 819us/sample - loss: 0.6848 - acc: 0.6119 - val_loss: 0.6854 - val_acc: 0.5648\n",
      "Epoch 14/100\n",
      "23750/23750 [==============================] - 20s 827us/sample - loss: 0.6828 - acc: 0.6248 - val_loss: 0.6833 - val_acc: 0.6080\n",
      "Epoch 15/100\n",
      "23750/23750 [==============================] - 18s 748us/sample - loss: 0.6807 - acc: 0.6442 - val_loss: 0.6812 - val_acc: 0.6448\n",
      "Epoch 16/100\n",
      "23168/23750 [============================>.] - ETA: 0s - loss: 0.6784 - acc: 0.6579- ETA: 1s - loss: 0.6784 - ac"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-9-216b3a98525a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m model.fit(x_train_pad, y_train,\n\u001b[1;32m----> 2\u001b[1;33m           validation_split=0.05, epochs=100, batch_size=64)\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[0;32m    878\u001b[0m           \u001b[0minitial_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    879\u001b[0m           \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 880\u001b[1;33m           validation_steps=validation_steps)\n\u001b[0m\u001b[0;32m    881\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    882\u001b[0m   def evaluate(self,\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training_arrays.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[1;34m(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, initial_epoch, steps_per_epoch, validation_steps, mode, validation_in_fit, **kwargs)\u001b[0m\n\u001b[0;32m    327\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    328\u001b[0m         \u001b[1;31m# Get outputs.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 329\u001b[1;33m         \u001b[0mbatch_outs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    330\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    331\u001b[0m           \u001b[0mbatch_outs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow\\python\\keras\\backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   3074\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3075\u001b[0m     fetched = self._callable_fn(*array_vals,\n\u001b[1;32m-> 3076\u001b[1;33m                                 run_metadata=self.run_metadata)\n\u001b[0m\u001b[0;32m   3077\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_fetch_callbacks\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfetched\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3078\u001b[0m     return nest.pack_sequence_as(self._outputs_structure,\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1437\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[0;32m   1438\u001b[0m               \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1439\u001b[1;33m               run_metadata_ptr)\n\u001b[0m\u001b[0;32m   1440\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1441\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.fit(x_train_pad, y_train,\n",
    "          validation_split=0.05, epochs=100, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = model.evaluate(x_test_pad, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Accuracy: {0:.2%}\".format(result[1]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py36",
   "language": "python",
   "name": "py36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}