tf_2.x/lab-12-4-many-to-many-bidirectional-keras-eager.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 12-4 many to many variable bidirectional\n",
    "### simple pos-tagger training \n",
    "* many to many\n",
    "* variable input sequence length\n",
    "* bi-directional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.0\n"
     ]
    }
   ],
   "source": [
    "# setup\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from pprint import pprint\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepairing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# example data\n",
    "sentences = [['I', 'feel', 'hungry'],\n",
    "     ['tensorflow', 'is', 'very', 'difficult'],\n",
    "     ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n",
    "     ['tensorflow', 'is', 'very', 'fast', 'changing']]\n",
    "pos = [['pronoun', 'verb', 'adjective'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective'],\n",
    "     ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective', 'verb']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'I': 1, 'a': 2, 'changing': 3, 'deep': 4, 'difficult': 5, 'fast': 6, 'feel': 7, 'for': 8, 'framework': 9, 'hungry': 10, 'is': 11, 'learning': 12, 'tensorflow': 13, 'very': 14}\n",
      "{0: '<pad>', 1: 'I', 2: 'a', 3: 'changing', 4: 'deep', 5: 'difficult', 6: 'fast', 7: 'feel', 8: 'for', 9: 'framework', 10: 'hungry', 11: 'is', 12: 'learning', 13: 'tensorflow', 14: 'very'}\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "# creating a token dictionary for word\n",
    "word_list = sum(sentences, [])\n",
    "word_list = sorted(set(word_list))\n",
    "word_list = ['<pad>'] + word_list\n",
    "word2idx = {word : idx for idx, word in enumerate(word_list)}\n",
    "idx2word = {idx : word for idx, word in enumerate(word_list)}\n",
    "\n",
    "print(word2idx)\n",
    "print(idx2word)\n",
    "print(len(idx2word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'adjective': 1, 'adverb': 2, 'determiner': 3, 'noun': 4, 'preposition': 5, 'pronoun': 6, 'verb': 7}\n",
      "{0: '<pad>', 1: 'adjective', 2: 'adverb', 3: 'determiner', 4: 'noun', 5: 'preposition', 6: 'pronoun', 7: 'verb'}\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "# creating a token dictionary for part of speech\n",
    "pos_list = sum(pos, [])\n",
    "pos_list = sorted(set(pos_list))\n",
    "pos_list = ['<pad>'] + pos_list\n",
    "pos2idx = {pos : idx for idx, pos in enumerate(pos_list)}\n",
    "idx2pos = {idx : pos for idx, pos in enumerate(pos_list)}\n",
    "\n",
    "print(pos2idx)\n",
    "print(idx2pos)\n",
    "print(len(pos2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1  7 10  0  0  0  0  0  0  0]\n",
      " [13 11 14  5  0  0  0  0  0  0]\n",
      " [13 11  2  9  8  4 12  0  0  0]\n",
      " [13 11 14  6  3  0  0  0  0  0]] [3, 4, 7, 5]\n",
      "[[1. 1. 1. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 1. 1. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]]\n",
      "[[6 7 1 0 0 0 0 0 0 0]\n",
      " [4 7 2 1 0 0 0 0 0 0]\n",
      " [4 7 3 4 5 1 4 0 0 0]\n",
      " [4 7 2 1 7 0 0 0 0 0]]\n"
     ]
    }
   ],
   "source": [
    "# converting sequence of tokens to sequence of indices\n",
    "max_sequence = 10\n",
    "x_data = list(map(lambda sentence : [word2idx.get(token) for token in sentence], sentences))\n",
    "y_data = list(map(lambda sentence : [pos2idx.get(token) for token in sentence], pos))\n",
    "\n",
    "# padding the sequence of indices\n",
    "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence, padding='post')\n",
    "x_data_mask = ((x_data != 0) * 1).astype(np.float32)\n",
    "x_data_len = list(map(lambda sentence : len(sentence), sentences))\n",
    "\n",
    "y_data = pad_sequences(sequences = y_data, maxlen = max_sequence, padding='post')\n",
    "\n",
    "# checking data\n",
    "print(x_data, x_data_len)\n",
    "print(x_data_mask)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating bidirectional rnn for \"many to many\" sequence tagging\n",
    "num_classes = len(pos2idx)\n",
    "hidden_dim = 10\n",
    "\n",
    "input_dim = len(word2idx)\n",
    "output_dim = len(word2idx)\n",
    "one_hot = np.eye(len(word2idx))\n",
    "\n",
    "model = Sequential()\n",
    "model.add(layers.InputLayer(input_shape=(max_sequence,)))\n",
    "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim, mask_zero=True,\n",
    "                                 trainable=False, input_length=max_sequence,\n",
    "                                 embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
    "model.add(layers.Bidirectional(keras.layers.SimpleRNN(units=hidden_dim, return_sequences=True)))\n",
    "model.add(layers.TimeDistributed(keras.layers.Dense(units=num_classes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 10, 15)            225       \n",
      "_________________________________________________________________\n",
      "bidirectional (Bidirectional (None, 10, 20)            520       \n",
      "_________________________________________________________________\n",
      "time_distributed (TimeDistri (None, 10, 8)             168       \n",
      "=================================================================\n",
      "Total params: 913\n",
      "Trainable params: 688\n",
      "Non-trainable params: 225\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating loss function\n",
    "def loss_fn(model, x, y, x_len, max_sequence):\n",
    "    masking = tf.sequence_mask(x_len, maxlen=max_sequence, dtype=tf.float32)\n",
    "    valid_time_step = tf.cast(x_len,dtype=tf.float32)\n",
    "    sequence_loss = tf.keras.losses.sparse_categorical_crossentropy(\n",
    "        y_true=y, y_pred=model(x), from_logits=True) * masking    \n",
    "    sequence_loss = tf.reduce_sum(sequence_loss, axis=-1) / valid_time_step\n",
    "    sequence_loss = tf.reduce_mean(sequence_loss)\n",
    "    return sequence_loss\n",
    "\n",
    "# creating and optimizer\n",
    "lr = 0.1\n",
    "epochs = 30\n",
    "batch_size = 2 \n",
    "opt = tf.keras.optimizers.Adam(learning_rate = lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((None, 10), (None, 10), (None,)), types: (tf.int32, tf.int32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "# generating data pipeline\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data, x_data_len))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size=4)\n",
    "tr_dataset = tr_dataset.batch(batch_size = 2)\n",
    "\n",
    "print(tr_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.014\n",
      "epoch :  10, tr_loss : 0.001\n",
      "epoch :  15, tr_loss : 0.000\n",
      "epoch :  20, tr_loss : 0.000\n",
      "epoch :  25, tr_loss : 0.000\n",
      "epoch :  30, tr_loss : 0.000\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "tr_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    avg_tr_loss = 0\n",
    "    tr_step = 0\n",
    "    \n",
    "    for x_mb, y_mb, x_mb_len in tr_dataset:\n",
    "        with tf.GradientTape() as tape:\n",
    "            tr_loss = loss_fn(model, x=x_mb, y=y_mb, x_len=x_mb_len, max_sequence=max_sequence)\n",
    "        grads = tape.gradient(target=tr_loss, sources=model.variables)\n",
    "        opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
    "        avg_tr_loss += tr_loss\n",
    "        tr_step += 1\n",
    "    else:\n",
    "        avg_tr_loss /= tr_step\n",
    "        tr_loss_hist.append(avg_tr_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 == 0:\n",
    "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch + 1, avg_tr_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['pronoun', 'verb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', 'verb', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']]\n",
      "[['pronoun', 'verb', 'adjective'],\n",
      " ['noun', 'verb', 'adverb', 'adjective'],\n",
      " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', 'verb']]\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_data)\n",
    "yhat = np.argmax(yhat, axis=-1) * x_data_mask\n",
    "\n",
    "pprint(list(map(lambda row : [idx2pos.get(elm) for elm in row],yhat.astype(np.int32).tolist())), width = 120)\n",
    "pprint(pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x2a0cf3eb9b0>]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAaX0lEQVR4nO3dfZRcd33f8fdnZnck7VjSzlgr27EeViYG7IItWWMlHNPE9IAjpwmClgYpoZgeQKXFJS2nPbWbHpuaNoeTtE0KphgBOoa2SHEMJsqpiDHBxDTEQSsjP8lPivDDIttaeyVbsh5399s/5q40Wu3D7O7sjubez+ucOTP3d3/3zu/qHn3m7u/e+7uKCMzMLP1yzW6AmZnNDge+mVlGOPDNzDLCgW9mlhEOfDOzjGhrdgNGs2jRouju7m52M8zMWsbOnTtfiYiu8eqck4Hf3d1NT09Ps5thZtYyJD03UR136ZiZZYQD38wsIxz4ZmYZ4cA3M8uICU/aStoM/AawPyLeNsr8fwf8Ts36LgO6IqJf0rPAIWAQGIiISqMabmZmk1PPEf6dwNqxZkbEH0bEyohYCdwM/FVE9NdUeVcy32FvZtZEEwZ+RDwA9E9UL7EB2DKtFpmZ2YxoWB++pA6qfwl8q6Y4gO9J2ilp4wTLb5TUI6mnr69v0t8fEXz+L5/hr56e/LJmZlnQyJO2vwn89YjunGsi4irgeuCTkn5lrIUjYlNEVCKi0tU17s1io5LEVx7Yy/1P7p/0smZmWdDIwF/PiO6ciNiXvO8H7gHWNPD7zlIqFjh45MRMfoWZWctqSOBLWgj8KvBnNWVFSfOHPwPXAY814vvGUioW6D9ycia/wsysZdVzWeYW4FpgkaRe4FagHSAi7kiqvR/4XkS8UbPoBcA9koa/55sR8ReNa/rZSh3tvHrYR/hmZqOZMPAjYkMdde6kevlmbdle4MqpNmwqyh0Fnnn58Gx+pZlZy0jVnbalYoED7sM3MxtVqgK/XCxw5MQgx04ONrspZmbnnFQFfqmjAMBBn7g1MztLygK/HYD+N9ytY2Y2UroCv1g9wnc/vpnZ2VIV+GUHvpnZmFIV+MN9+AfcpWNmdpZUBX7nqT58n7Q1MxspVYHfns8xf26bu3TMzEaRqsCHaj++r9IxMztb6gK/1OG7bc3MRpO6wC97eAUzs1GlLvA7O9o54JO2ZmZnSV3glzvch29mNprUBX6pWODoSQ+gZmY2UuoC33fbmpmNLnWB7wHUzMxGl8LAHx5ewSduzcxqpS7wh7t0+t2lY2Z2hgkDX9JmSfslPTbG/GslvSZpV/K6pWbeWklPSdoj6aZGNnwsw0MkH3Tgm5mdoZ4j/DuBtRPU+VFErExetwFIygNfBK4HLgc2SLp8Oo2tR+c89+GbmY1mwsCPiAeA/imsew2wJyL2RsQJYCuwbgrrmZS2fI4Fc9s8RLKZ2QiN6sN/h6SHJX1X0t9Lyi4GXqip05uUjUrSRkk9knr6+vqm1ZhysUC/n2trZnaGRgT+Q8DyiLgS+ALwnaRco9SNsVYSEZsiohIRla6urmk1qFQs+AjfzGyEaQd+RLweEYeTz9uBdkmLqB7RL62pugTYN93vq0fZI2aamZ1l2oEv6UJJSj6vSdb5KrADuFTSCkkFYD2wbbrfVw8f4ZuZna1togqStgDXAosk9QK3Au0AEXEH8AHgX0gaAI4C6yMigAFJNwL3Anlgc0Q8PiNbMUKpo93X4ZuZjTBh4EfEhgnm3w7cPsa87cD2qTVt6krFAsdODnH0xCDzCvnZ/nozs3NS6u60hWofPngANTOzWqkM/OG7bX3zlZnZaekMfB/hm5mdJZWBXy56eAUzs5FSGfinh0h24JuZDUtl4C+c144EBzy8gpnZKakM/LZ8joXz2t2Hb2ZWI5WBD9VuHffhm5mdluLA9xG+mVmt1AZ+uVjwc23NzGqkNvBLHjHTzOwM6Q38YrUPvzqOm5mZpTfwOwocHxji6MnBZjfFzOyckNrA9922ZmZnSm3gD99te9A3X5mZASkO/LJHzDQzO0NqA7/TI2aamZ0htYHvI3wzszOlNvA9gJqZ2ZlSG/j5nOic1+4hks3MEhMGvqTNkvZLemyM+b8j6ZHk9WNJV9bMe1bSo5J2SeppZMPrUeoo0O8+fDMzoL4j/DuBtePM/xnwqxFxBfBZYNOI+e+KiJURUZlaE6euVCz4CN/MLDFh4EfEA0D/OPN/HBEHkskHgSUNatu0eYhkM7PTGt2H/1HguzXTAXxP0k5JG8dbUNJGST2Sevr6+hrSmHKx3TdemZkl2hq1Iknvohr476wpviYi9klaDNwn6cnkL4azRMQmku6gSqXSkBHPSsVqH35EIKkRqzQza1kNOcKXdAXwVWBdRLw6XB4R+5L3/cA9wJpGfF+9Sh0FTgwMceSEB1AzM5t24EtaBnwb+KcR8XRNeVHS/OHPwHXAqFf6zJRyh2++MjMbNmGXjqQtwLXAIkm9wK1AO0BE3AHcApwP/M+k22QguSLnAuCepKwN+GZE/MUMbMOYSsXTwyssLXfM5lebmZ1zJgz8iNgwwfyPAR8bpXwvcOXZS8ye4SGSfbetmVmK77SFmgHU3KVjZpbuwHcfvpnZaakO/AXz2snJQySbmUHKAz+fE50dBQe+mRkpD3yAUkc7B97wSVszswwEvsfTMTODLAR+0V06ZmaQgcAv+wjfzAzIQOCXigUOHjlJREPGYzMza1mpD/xysZ0Tg0O84QHUzCzjUh/4vtvWzKwq9YHvu23NzKpSH/i1I2aamWVZ6gO/7MA3MwMyEPiljuoQyf2+29bMMi71gb9gbjKAmvvwzSzjUh/4uZyqwyu4S8fMMi71gQ/DN1858M0s2zIR+B5ewcyszsCXtFnSfkmPjTFfkj4vaY+kRyRdVTPvBknPJK8bGtXwyej0EMlmZnUf4d8JrB1n/vXApclrI/AlAEll4Fbgl4A1wK2SSlNt7FSVi+7DNzOrK/Aj4gGgf5wq64BvRNWDQKeki4BfA+6LiP6IOADcx/g/HDNiuA/fA6iZWZY1qg//YuCFmunepGys8rNI2iipR1JPX19fg5pVVe4ocHIwOHx8oKHrNTNrJY0KfI1SFuOUn10YsSkiKhFR6erqalCzqjqTm6/cj29mWdaowO8FltZMLwH2jVM+q4aHV3A/vpllWaMCfxvw4eRqnV8GXouIF4F7gesklZKTtdclZbPq1ABqvjTTzDKsrZ5KkrYA1wKLJPVSvfKmHSAi7gC2A78O7AGOAP8smdcv6bPAjmRVt0XEeCd/Z8TwEMkeQM3MsqyuwI+IDRPMD+CTY8zbDGyefNMaZ/gI3zdfmVmWZeJO2wVz28jn5CN8M8u0TAS+JEod7R4i2cwyLROBD1DqKPikrZllWnYCv1hwl46ZZVp2Ar+j3YFvZpmWmcAvFwvuwzezTMtM4Jc6PICamWVbZgK/XCwwMBQc8gBqZpZRmQn8UoeHVzCzbMtO4BerI2b6blszy6rsBL7H0zGzjMtM4J8aItlX6phZRmUm8IcHUDvoI3wzy6jMBP78OW205eQ+fDPLrMwEviQ6Ozy8gpllV2YCH6BcbPcRvpllVqYCv9RR4MARn7Q1s2zKVOCXix4i2cyyK1OB7z58M8uyugJf0lpJT0naI+mmUeb/kaRdyetpSQdr5g3WzNvWyMZPVrnYzoEjJxka8gBqZpY9Ez7EXFIe+CLwHqAX2CFpW0TsHq4TEf+mpv6/AlbVrOJoRKxsXJOnrtRRYHAoOHRsgIUd7c1ujpnZrKrnCH8NsCci9kbECWArsG6c+huALY1oXKMN323rbh0zy6J6Av9i4IWa6d6k7CySlgMrgB/UFM+V1CPpQUnvm3JLG2D4btt+B76ZZdCEXTqARikbqxN8PXB3RAzWlC2LiH2SLgF+IOnRiPi7s75E2ghsBFi2bFkdzZo8D5FsZllWzxF+L7C0ZnoJsG+MuusZ0Z0TEfuS973ADzmzf7+23qaIqEREpaurq45mTV65Y3gANQe+mWVPPYG/A7hU0gpJBaqhftbVNpLeApSAv6kpK0mak3xeBFwD7B657GwZHhP/oG++MrMMmrBLJyIGJN0I3Avkgc0R8bik24CeiBgO/w3A1jjzobGXAV+WNET1x+VztVf3zLbz5rTRnpf78M0sk+rpwycitgPbR5TdMmL6M6Ms92Pg7dNoX0OdGkDNXTpmlkGZutMWqv347sM3syzKXOCXiu2+Dt/MMilzgV8uesRMM8umzAV+yX34ZpZR2Qz8Iyc8gJqZZU72Ar9YYCjg9WPu1jGzbMlc4JeTm698pY6ZZU3mAv/UeDo+cWtmGZPdwPcRvpllTOYCv+whks0sozIX+MNj4vsI38yyJnOBXyzkKeRz7sM3s8zJXOBLqg6v4CN8M8uYzAU+VE/cug/fzLIms4HvI3wzy5pMBn656CN8M8ueTAZ+qdjuxxyaWeZkM/CTAdQGPYCamWVIJgN/WbmDCNjbd7jZTTEzmzWZDPxKdxmAnucONLklZmazp67Al7RW0lOS9ki6aZT5H5HUJ2lX8vpYzbwbJD2TvG5oZOOnqvv8Ds4vFuh51oFvZtnRNlEFSXngi8B7gF5gh6RtEbF7RNU/iYgbRyxbBm4FKkAAO5Nlm5q0kli9vETPc/3NbIaZ2ayq5wh/DbAnIvZGxAlgK7CuzvX/GnBfRPQnIX8fsHZqTW2sq7vLPPfqEfYfOtbsppiZzYp6Av9i4IWa6d6kbKR/LOkRSXdLWjrJZZG0UVKPpJ6+vr46mjU9q7tLAOx0t46ZZUQ9ga9RykZez/jnQHdEXAF8H/j6JJatFkZsiohKRFS6urrqaNb0vO0XFjKnLecTt2aWGfUEfi+wtGZ6CbCvtkJEvBoRx5PJrwCr6122WQptOa5c2unAN7PMqCfwdwCXSlohqQCsB7bVVpB0Uc3ke4Enks/3AtdJKkkqAdclZeeEyvISj//8NY6eGGx2U8zMZtyEgR8RA8CNVIP6CeCuiHhc0m2S3ptU+5SkxyU9DHwK+EiybD/wWao/GjuA25Kyc8LV3WUGhoJdLxxsdlPMzGbchJdlAkTEdmD7iLJbaj7fDNw8xrKbgc3TaOOMuWpZcuL2uX7e8abzm9waM7OZlck7bYct7GjnzRecxw5fqWNmGZDpwIfqMAsPPX+AIQ+kZmYp58BfXuLQsQGe3n+o2U0xM5tRDvzl1YHU3K1jZmmX+cBfWp7H4vlz2PnsOXPxkJnZjMh84Eui0l3yEb6ZpV7mAx9g9fIyPz94lJde80BqZpZeDnzg6mQgNQ+XbGZp5sAHLrtoAfPa834gipmlmgMfaM/nWLm000f4ZpZqDvzE1d0ldu97ncPHB5rdFDOzGeHAT6zuLjMUsOt5D6RmZunkwE9ctayTnHzi1szSy4GfmD+3nbdcuMAnbs0stRz4NSrLS/z0+QMMDA41uylmZg3nwK9R6S7xxolBnnzJA6mZWfo48GtUuqsDqfV4XB0zSyEHfo2LO+dx0cK5frC5maWSA3+ESneZnmcPEOEHophZutQV+JLWSnpK0h5JN40y/9OSdkt6RNJfSlpeM29Q0q7kta2RjZ8JleUlXnr9GD8/eLTZTTEza6gJA19SHvgicD1wObBB0uUjqv0UqETEFcDdwB/UzDsaESuT13sb1O4Zs3r58IPN3a1jZulSzxH+GmBPROyNiBPAVmBdbYWIuD8ijiSTDwJLGtvM2fPWC+dz3pw2dvjErZmlTD2BfzHwQs10b1I2lo8C362ZniupR9KDkt431kKSNib1evr6+upo1sxoy+dYtazTN2CZWerUE/gapWzUM5qSPgRUgD+sKV4WERXgt4E/lvSm0ZaNiE0RUYmISldXVx3Nmjmrl5d46uVDvH7sZFPbYWbWSPUEfi+wtGZ6CbBvZCVJ7wZ+D3hvRBwfLo+Ifcn7XuCHwKpptHdWXN1dJgIecj++maVIPYG/A7hU0gpJBWA9cMbVNpJWAV+mGvb7a8pLkuYknxcB1wC7G9X4mbJyaSf5nHzi1sxSpW2iChExIOlG4F4gD2yOiMcl3Qb0RMQ2ql045wF/Kgng+eSKnMuAL0saovrj8rmIOOcDvzinjcsumu9+fDNLlQkDHyAitgPbR5TdUvP53WMs92Pg7dNpYLNUlpfZuuN5Tg4O0Z73/Wlm1vqcZGOodJc4dnKI3fteb3ZTzMwawoE/hsry6kBqvh7fzNLCgT+GCxfOZUlpnk/cmllqOPDHUVleouc5D6RmZungwB9HpbtM36HjPN9/ZOLKZmbnOAf+OCrd1YHUfHmmmaWBA38cb148nwsWzOH2+/fw2hEPs2Bmrc2BP45cTnxhw1X0HjjCjVse8sPNzaylOfAnsGZFmc+uexs/euYVfn/7k81ujpnZlNV1p23WrV+zjCdfOsTmv/4Zb71wPr919dKJFzIzO8f4CL9O//EfXsY7f3ERv/edR+nxzVhm1oIc+HVqy+e4/bdXcXHnPD7xv3f6mbdm1nIc+JPQ2VHgqzdUOH5yiI9/vYcjJwaa3SQzs7o58CfpFxfP5/MbVvHES6/zb//0YYaGfBeumbUGB/4UvOuti7n5+rey/dGX+MIP9jS7OWZmdfFVOlP08b9/CU++eIg/+v7TvPmC87j+7Rc1u0lmZuPyEf4USeL3/9HbWbWsk0/f9bDHzTezc54Dfxrmtuf58odWs3BeOx//Rg+vHD4+8UJmZk3iwJ+mxQvmsunDq3nl8HE+/LWf8NUf7WXnc/0cOznY7KaZmZ3BffgNcMWSTj6/YRW3/flu/vP/fQKAtpy47KIFrFzaWX0t62TF+UVyOTW5tWaWVarn4R6S1gL/A8gDX42Iz42YPwf4BrAaeBX4YEQ8m8y7GfgoMAh8KiLunej7KpVK9PT0TG5LzhH7Dx1j1/MH2fXCQX76/EEe6T3IGyeqR/sL5rZx5dJOVi3tpHtRkcXz57J4wRwWz5/DwnntSP4xMLOpkbQzIirj1pko8CXlgaeB9wC9wA5gQ0TsrqnzL4ErIuITktYD74+ID0q6HNgCrAF+Afg+8OaIGLe/o5UDf6TBoWDP/sPseuHAqR+Bp18+xMjL9wv5HF3z59A1v/oDUH2fS7nYTls+R14inxNteZGTaMuJXK76nq95tedz1fdc8p4/s7wtL9py1fXlcpDPnV5fPif/6Ji1qHoCv54unTXAnojYm6x0K7AO2F1TZx3wmeTz3cDtqibHOmBrRBwHfiZpT7K+v5nMhrSyfE685cL5vOXC+Xzw6mUAHD0xyIuvHaXv0HH2J6/q52OnnrDV89wB+t84MevtlUh+DFR9V/WKJAEIRDJd+zlZrlrCqXmcKgehms+133f2D0xt0Wi/P2L8ZU7XG2sb6/tRm9RP3wz8Tjbzp9c//I1V779mqaPAXZ94x4y1o57Avxh4oWa6F/ilsepExICk14Dzk/IHRyx78WhfImkjsBFg2bJl9bS9Zc0r5Lmk6zwu6Tpv3HonBoZ47ehJhiIYGAoGB4PBCAaHhhgYCgYG4/S8ZHqgZl5tver70Kn3wYChoeH1BUND1TpDyfRgVMuGAiIgCIb/GIwIgjPLh6eTGkk9Tr3HiLLTtUaWjVKBcYtGfebwWH+31vt44sncPz0Tzzxu6v3bvnm8oWIS/6AL5rbPYEvqC/zRfpxGbsFYdepZtloYsQnYBNUunTralXqFtmo3j5lZI9RzWWYvUDsA/BJg31h1JLUBC4H+Opc1M7NZUE/g7wAulbRCUgFYD2wbUWcbcEPy+QPAD6L6d+42YL2kOZJWAJcCP2lM083MbDIm7NJJ+uRvBO6lelnm5oh4XNJtQE9EbAO+Bvyv5KRsP9UfBZJ6d1E9wTsAfHKiK3TMzGxm1HUd/mxL02WZZmazoZ7LMj20gplZRjjwzcwywoFvZpYRDnwzs4w4J0/aSuoDnpvi4ouAVxrYnGZL2/ZA+rYpbdsD6dumtG0PnL1NyyOia7wFzsnAnw5JPROdqW4ladseSN82pW17IH3blLbtgaltk7t0zMwywoFvZpYRaQz8Tc1uQIOlbXsgfduUtu2B9G1T2rYHprBNqevDNzOz0aXxCN/MzEbhwDczy4jUBL6ktZKekrRH0k3Nbk8jSHpW0qOSdklqydHkJG2WtF/SYzVlZUn3SXomeS81s42TMcb2fEbSz5P9tEvSrzezjZMhaamk+yU9IelxSb+blLfyPhprm1pyP0maK+knkh5Otuc/JeUrJP1tso/+JBm+fvx1paEPv54HrbciSc8ClYho2RtGJP0KcBj4RkS8LSn7A6A/Ij6X/DiXIuLfN7Od9Rpjez4DHI6I/9rMtk2FpIuAiyLiIUnzgZ3A+4CP0Lr7aKxt+i1acD8lzwcvRsRhSe3A/wN+F/g08O2I2CrpDuDhiPjSeOtKyxH+qQetR8QJYPhB69ZkEfEA1Wck1FoHfD35/HWq/xlbwhjb07Ii4sWIeCj5fAh4gupzp1t5H421TS0pqg4nk+3JK4B/ANydlNe1j9IS+KM9aL1ld3CNAL4naWfykPe0uCAiXoTqf05gcZPb0wg3Snok6fJpme6PWpK6gVXA35KSfTRim6BF95OkvKRdwH7gPuDvgIMRMZBUqSvz0hL4dT8svcVcExFXAdcDn0y6E+zc8yXgTcBK4EXgvzW3OZMn6TzgW8C/jojXm92eRhhlm1p2P0XEYESspPpc8DXAZaNVm2g9aQn8VD4sPSL2Je/7gXuo7ug0eDnpZx3ub93f5PZMS0S8nPyHHAK+Qovtp6Rf+FvA/4mIbyfFLb2PRtumVt9PABFxEPgh8MtAp6Thx9TWlXlpCfx6HrTeUiQVkxNOSCoC1wGPjb9Uy6h96P0NwJ81sS3TNhyMiffTQvspOSH4NeCJiPjvNbNadh+NtU2tup8kdUnqTD7PA95N9bzE/cAHkmp17aNUXKUDkFxi9cecftD6f2lyk6ZF0iVUj+qh+rD5b7biNknaAlxLdSjXl4Fbge8AdwHLgOeBfxIRLXEidIztuZZqN0EAzwL/fLj/+1wn6Z3Aj4BHgaGk+D9Q7fNu1X001jZtoAX3k6QrqJ6UzVM9SL8rIm5LMmIrUAZ+CnwoIo6Pu660BL6ZmY0vLV06ZmY2AQe+mVlGOPDNzDLCgW9mlhEOfDOzjHDgm5llhAPfzCwj/j/fC2wHtCEGvwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}