Day4_2_Ma_LSTM.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Day4_2_Ma_LSTM.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"markdown","metadata":{"id":"bG4_-K7Z6aub","colab_type":"text"},"source":["# Ma LSTM"]},{"cell_type":"code","metadata":{"id":"ixJi5iWk6VzH","colab_type":"code","colab":{}},"source":["import sys\n","import tensorflow as tf\n","import numpy as np\n","import os\n","import pandas as pd\n","\n","from sklearn.model_selection import train_test_split\n","\n","import json"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"x2ozbQUx6crE","colab_type":"code","outputId":"4f705b94-0178-40e9-8111-af3f58391a19","executionInfo":{"status":"ok","timestamp":1561176443209,"user_tz":-540,"elapsed":5888,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":54}},"source":["from google.colab import auth\n","auth.authenticate_user()\n","\n","from google.colab import drive\n","drive.mount('/content/gdrive')"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"q1c3mIopN57-","colab_type":"code","colab":{}},"source":["DATA_IN_PATH = './data_in/'\n","DATA_OUT_PATH = './data_out/'"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"4ng1WcFqu2Tr","colab_type":"code","outputId":"120f45a9-61b9-4595-c0ce-2f05767e8a4f","executionInfo":{"status":"ok","timestamp":1561176444818,"user_tz":-540,"elapsed":4918,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":168}},"source":["!ls -al"],"execution_count":0,"outputs":[{"output_type":"stream","text":["total 40\n","drwxr-xr-x 1 root root 4096 Jun 22 03:59 .\n","drwxr-xr-x 1 root root 4096 Jun 22 03:16 ..\n","-rw-r--r-- 1 root root 2487 Jun 22 03:21 adc.json\n","drwxr-xr-x 1 root root 4096 Jun 22 03:21 .config\n","drwxr-xr-x 2 root root 4096 Jun 22 04:04 data_in\n","drwxr-xr-x 2 root root 4096 Jun 22 03:59 data_out\n","drwx------ 3 root root 4096 Jun 22 03:21 gdrive\n","drwxr-xr-x 1 root root 4096 Jun 18 16:14 sample_data\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"XqBTKs0MN9s2","colab_type":"code","colab":{}},"source":["TRAIN_Q1_DATA_FILE = 'train_q1.npy'\n","TRAIN_Q2_DATA_FILE = 'train_q2.npy'\n","TRAIN_LABEL_DATA_FILE = 'train_label.npy'\n","NB_WORDS_DATA_FILE = 'data_configs.json'\n","\n","## 학습에 필요한 파라메터들에 대해서 지정하는 부분이다.\n","\n","BATCH_SIZE = 4096\n","EPOCH = 2\n","HIDDEN = 64\n","\n","DROPOUT_RATIO = 0.3\n","\n","TEST_SPLIT = 0.1\n","RNG_SEED = 13371447\n","EMBEDDING_DIM = 128\n","MAX_SEQ_LEN = 31"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"SMqbutxbOBjN","colab_type":"code","colab":{}},"source":["## 데이터를 불러오는 부분이다. 효과적인 데이터 불러오기를 위해, 미리 넘파이 형태로 저장시킨 데이터를 로드한다.\n","\n","q1_data = np.load(open(DATA_IN_PATH + TRAIN_Q1_DATA_FILE, 'rb'))\n","q2_data = np.load(open(DATA_IN_PATH + TRAIN_Q2_DATA_FILE, 'rb'))\n","labels = np.load(open(DATA_IN_PATH + TRAIN_LABEL_DATA_FILE, 'rb'))\n","prepro_configs = None\n","\n","with open(DATA_IN_PATH + NB_WORDS_DATA_FILE, 'r') as f:\n","    prepro_configs = json.load(f)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"kgEEPYtIOD5E","colab_type":"code","outputId":"5686af84-d2cd-4a6f-b4ba-a3202d23d5f0","executionInfo":{"status":"ok","timestamp":1561176502773,"user_tz":-540,"elapsed":555,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["VOCAB_SIZE = prepro_configs['vocab_size']\n","VOCAB_SIZE"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["76364"]},"metadata":{"tags":[]},"execution_count":7}]},{"cell_type":"markdown","metadata":{"id":"TyIcRnSAwA75","colab_type":"text"},"source":["### Split train and test dataset"]},{"cell_type":"code","metadata":{"id":"nrdXZS1XOFm8","colab_type":"code","colab":{}},"source":["q1_data_len = np.array([min(len(x), MAX_SEQ_LEN) for x in q1_data])\n","q2_data_len = np.array([min(len(x), MAX_SEQ_LEN) for x in q2_data])"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"3840A8nNOPUl","colab_type":"code","colab":{}},"source":["## 데이터를 나누어 저장하자. sklearn의 train_test_split을 사용하면 유용하다. 하지만, 쿼라 데이터의 경우는\n","## 입력이 1개가 아니라 2개이다. 따라서, np.stack을 사용하여 두개를 하나로 쌓은다음 활용하여 분류한다.\n","\n","X = np.stack((q1_data, q2_data), axis=1)\n","y = labels\n","train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=TEST_SPLIT, random_state=RNG_SEED)\n","\n","train_Q1 = train_X[:,0]\n","train_Q2 = train_X[:,1]\n","test_Q1 = test_X[:,0]\n","test_Q2 = test_X[:,1]"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Orbt7YELOQ91","colab_type":"code","colab":{}},"source":["\n","def rearrange(base, hypothesis, labels):\n","    features = {\"base\": base, \"hypothesis\": hypothesis}\n","    return features, labels\n","\n","def train_input_fn():\n","    dataset = tf.data.Dataset.from_tensor_slices((train_Q1, train_Q2, train_y))\n","    dataset = dataset.shuffle(buffer_size=len(train_Q1))\n","    dataset = dataset.batch(BATCH_SIZE) #4096\n","    dataset = dataset.map(rearrange)\n","    dataset = dataset.repeat(EPOCH) # 2\n","    iterator = dataset.make_one_shot_iterator()\n","    \n","    return iterator.get_next()\n","\n","def eval_input_fn():\n","    dataset = tf.data.Dataset.from_tensor_slices((test_Q1, test_Q2, test_y))\n","    dataset = dataset.batch(BATCH_SIZE) #4096\n","    dataset = dataset.map(rearrange)\n","    iterator = dataset.make_one_shot_iterator()\n","    \n","    return iterator.get_next()"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"nvXOXlSywPR2","colab_type":"text"},"source":["### Model Setup"]},{"cell_type":"code","metadata":{"colab_type":"code","id":"SROF7Q--JsQW","colab":{}},"source":["def Malstm(features, labels, mode):\n","        \n","    TRAIN = mode == tf.estimator.ModeKeys.TRAIN\n","    EVAL = mode == tf.estimator.ModeKeys.EVAL\n","    PREDICT = mode == tf.estimator.ModeKeys.PREDICT\n","    # VOCAB_SIZE : 76558   EMBEDDING_DIM : 128       \n","    embedding = tf.keras.layers.Embedding(VOCAB_SIZE, EMBEDDING_DIM) \n","    \n","    base_embedded_matrix = embedding(features['base']) # (?, 31, 128)\n","    \n","    hypothesis_embedded_matrix = embedding(features['hypothesis']) # (?, 31, 128)\n","    \n","    q_lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units = HIDDEN, activation = tf.nn.tanh , state_is_tuple=False)\n","    \n","    q, q_output_states = tf.nn.dynamic_rnn(cell = q_lstm_cell, #  LSTMStateTuple(c(?, 64), h(?, 64))\n","                                           inputs = base_embedded_matrix, # shape=(?, 128)\n","                                           dtype = tf.float32,\n","                                           scope='query')       \n","    \n","    s_lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units = HIDDEN, activation = tf.nn.tanh, state_is_tuple=False) \n","\n","    s, s_output_states = tf.nn.dynamic_rnn(cell = s_lstm_cell, \n","                                           inputs = hypothesis_embedded_matrix, \n","                                           dtype = tf.float32,\n","                                           scope='sim_query')     \n","\n","    with tf.variable_scope('output_layer'):\n","        logit_layer = tf.exp(-tf.reduce_sum(tf.abs(q_output_states - s_output_states), axis=1, keepdims=True)) # (?, 1)\n","        logit_layer = tf.squeeze(logit_layer, axis=-1) #(?,)\n","      \n","    if PREDICT:\n","        return tf.estimator.EstimatorSpec(\n","                  mode=mode,\n","                  predictions={\n","                      'is_duplicate':logit_layer\n","                  })\n","    \n","    #prediction 진행 시, None\n","    if labels is not None:\n","        labels = tf.to_float(labels)\n","    \n","    loss = tf.losses.mean_squared_error(labels=labels, predictions=logit_layer)\n","\n","    if EVAL:\n","        accuracy = tf.metrics.accuracy(labels, tf.round(logit_layer))\n","        eval_metric_ops = {'acc': accuracy}\n","        return tf.estimator.EstimatorSpec(\n","                  mode=mode,\n","                  eval_metric_ops= eval_metric_ops,\n","                  loss=loss)\n","\n","    elif TRAIN:\n","\n","        global_step = tf.train.get_global_step()\n","        train_op = tf.train.AdamOptimizer(1e-3).minimize(loss, global_step)\n","\n","        return tf.estimator.EstimatorSpec(\n","                  mode=mode,\n","                  train_op=train_op,\n","                  loss=loss)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"jnCkOq-BKi9Y","colab_type":"code","colab":{}},"source":["!rm -rf /checkpoint/malstm/"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"jBYLAo6dQIWF","colab_type":"code","colab":{}},"source":["model_dir = os.path.join(os.getcwd(), DATA_OUT_PATH + \"/checkpoint/malstm/\")\n","os.makedirs(model_dir, exist_ok=True)\n","\n","config_tf = tf.estimator.RunConfig()\n","\n","lstm_est = tf.estimator.Estimator(Malstm, model_dir=model_dir)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"NYXw_VqWeIW8","colab_type":"text"},"source":["### 아래 루틴 엄청 오래 걸림"]},{"cell_type":"code","metadata":{"id":"I_RfzgupQMSl","colab_type":"code","outputId":"7a290dac-aeae-42f4-8596-87a606d11f59","executionInfo":{"status":"ok","timestamp":1561177999270,"user_tz":-540,"elapsed":369901,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":558}},"source":["lstm_est.train(train_input_fn)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["WARNING: Logging before flag parsing goes to stderr.\n","W0622 04:27:09.665916 140227660601216 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.\n","W0622 04:27:10.033512 140227660601216 deprecation.py:323] From <ipython-input-12-06a9cfe2a426>:12: DatasetV1.make_one_shot_iterator (from tensorflow.python.data.ops.dataset_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use `for ... in dataset:` to iterate over a dataset. If using `tf.estimator`, return the `Dataset` object directly from your input function. As a last resort, you can use `tf.compat.v1.data.make_one_shot_iterator(dataset)`.\n","W0622 04:27:10.858126 140227660601216 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/initializers.py:119: calling RandomUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Call initializer instance with the dtype argument instead of passing it to the constructor\n","W0622 04:27:10.887599 140227660601216 deprecation.py:323] From <ipython-input-13-618ace763c19>:13: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n","W0622 04:27:10.889256 140227660601216 rnn_cell_impl.py:697] <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f8909fcb160>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","W0622 04:27:10.891503 140227660601216 deprecation.py:323] From <ipython-input-13-618ace763c19>:18: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n","W0622 04:27:11.182460 140227660601216 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Call initializer instance with the dtype argument instead of passing it to the constructor\n","W0622 04:27:11.197704 140227660601216 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn_cell_impl.py:738: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Call initializer instance with the dtype argument instead of passing it to the constructor\n","W0622 04:27:11.652121 140227660601216 rnn_cell_impl.py:697] <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f8909f6fef0>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","W0622 04:27:11.783504 140227660601216 deprecation.py:323] From <ipython-input-13-618ace763c19>:40: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use `tf.cast` instead.\n","W0622 04:27:11.801428 140227660601216 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/losses/losses_impl.py:121: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.where in 2.0, which has the same broadcast rule as np.where\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["<tensorflow_estimator.python.estimator.estimator.Estimator at 0x7f892a079358>"]},"metadata":{"tags":[]},"execution_count":16}]},{"cell_type":"code","metadata":{"id":"Lf5pVJ2mQOH4","colab_type":"code","outputId":"8d9ecbca-4430-4a24-e0e3-01efe1f48003","executionInfo":{"status":"ok","timestamp":1561178010095,"user_tz":-540,"elapsed":354196,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":138}},"source":["lstm_est.evaluate(eval_input_fn)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["W0622 04:33:19.597750 140227660601216 rnn_cell_impl.py:697] <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f8902396208>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","W0622 04:33:19.720473 140227660601216 rnn_cell_impl.py:697] <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f8909a3bf60>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","W0622 04:33:19.999620 140227660601216 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use standard file APIs to check for files with this prefix.\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["{'acc': 0.71852076, 'global_step': 132, 'loss': 0.18716064}"]},"metadata":{"tags":[]},"execution_count":17}]},{"cell_type":"code","metadata":{"id":"GLab41oLQSQV","colab_type":"code","colab":{}},"source":["TEST_Q1_DATA_FILE = 'test_q1.npy'\n","TEST_Q2_DATA_FILE = 'test_q2.npy'\n","TEST_ID_DATA_FILE = 'test_id.npy'\n","\n","test_q1_data = np.load(open(DATA_IN_PATH + TEST_Q1_DATA_FILE, 'rb'))\n","test_q2_data = np.load(open(DATA_IN_PATH + TEST_Q2_DATA_FILE, 'rb'))\n","test_id_data = np.load(open(DATA_IN_PATH + TEST_ID_DATA_FILE, 'rb'))"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"ky3XTfebQWeR","colab_type":"code","outputId":"fb94ade2-107d-48ad-d1fd-787c42ec97e7","executionInfo":{"status":"ok","timestamp":1561178743369,"user_tz":-540,"elapsed":1065110,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":222}},"source":["predict_input_fn = tf.estimator.inputs.numpy_input_fn(x={\"base\":test_q1_data, \n","                                                         \"hypothesis\":test_q2_data}, \n","                                                      shuffle=False)\n","\n","predictions = np.array([p['is_duplicate'] for p in lstm_est.predict(input_fn=predict_input_fn)])"],"execution_count":0,"outputs":[{"output_type":"stream","text":["W0622 04:33:51.712189 140227660601216 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_estimator/python/estimator/inputs/queues/feeding_queue_runner.py:62: QueueRunner.__init__ (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","To construct input pipelines, use the `tf.data` module.\n","W0622 04:33:51.738292 140227660601216 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_estimator/python/estimator/inputs/queues/feeding_functions.py:500: add_queue_runner (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","To construct input pipelines, use the `tf.data` module.\n","W0622 04:33:51.793607 140227660601216 rnn_cell_impl.py:697] <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f89077446a0>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","W0622 04:33:52.140244 140227660601216 rnn_cell_impl.py:697] <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f890772fa90>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","W0622 04:33:52.826818 140227660601216 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/monitored_session.py:875: start_queue_runners (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","To construct input pipelines, use the `tf.data` module.\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"mU3yVFv2Qbs2","colab_type":"code","outputId":"65677e2b-3b74-4bb2-d341-d3701548c0d7","executionInfo":{"status":"ok","timestamp":1561178751878,"user_tz":-540,"elapsed":1071623,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["print(len(predictions)) #2345796\n","\n","output = pd.DataFrame( data={\"test_id\":test_id_data, \"is_duplicate\": list(predictions)} )\n","output.to_csv( \"/content/gdrive/My Drive/Colab Notebooks/Data/rnn_predict.csv\", index=False, quoting=3 )"],"execution_count":0,"outputs":[{"output_type":"stream","text":["2345796\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"TqGo5SpXBSrX","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]}]}