Transformer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "class PositionalEncoding(tf.keras.layers.Layer):\n",
    "    def __init__(self, position, d_model):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        self.pos_encoding = self.positional_encoding(position, d_model)\n",
    "\n",
    "    def get_angles(self, position, i, d_model):\n",
    "        angles = 1 / tf.pow(10000, (2 * (i // 2)) / tf.cast(d_model, tf.float32))\n",
    "        return position * angles\n",
    "\n",
    "    def positional_encoding(self, position, d_model):\n",
    "        angle_rads = self.get_angles(\n",
    "            position=tf.range(position, dtype=tf.float32)[:, tf.newaxis],\n",
    "            i=tf.range(d_model, dtype=tf.float32)[tf.newaxis, :],\n",
    "            d_model=d_model)\n",
    "\n",
    "        # 배열의 짝수 인덱스(2i)에는 사인 함수 적용\n",
    "        sines = tf.math.sin(angle_rads[:, 0::2])\n",
    "\n",
    "        # 배열의 홀수 인덱스(2i+1)에는 코사인 함수 적용\n",
    "        cosines = tf.math.cos(angle_rads[:, 1::2])\n",
    "\n",
    "        pos_encoding = tf.concat([sines, cosines], axis=-1)\n",
    "        pos_encoding = pos_encoding[tf.newaxis, ...]\n",
    "        print(pos_encoding.shape)\n",
    "        return tf.cast(pos_encoding, tf.float32)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        return inputs + self.pos_encoding[:, :tf.shape(inputs)[1], :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 50, 128)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEKCAYAAAD+XoUoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3xcZZ348c/3nJnJPWnT9Jq2tJQCbRFKLQUscoflohZccUHBul4qu6KArIj60911fe2Ku66uK4JFq7gqLAhIwSK3BQsoQgu0tFzaUkqbNk2apk2aZJK5fX9/nJN0Ok0ykyaT5CTf9+v1vObMmXPOPA+XZ558n5uoKsYYY0YHZ6gzYIwxZvBYpW+MMaOIVfrGGDOKWKVvjDGjiFX6xhgzililb4wxo0heK30R2SYir4nIqyKyxj9XKSJPiMhm/3VsPvNgjDFDSURWiEi9iGzo4XMRkR+KyBYRWS8iC9I+u0hE3vI/u2Ug8jMYLf1zVHW+qi70398CPKWqs4Gn/PfGGDNS/QK4qJfPLwZm+2kZcDuAiLjAbf7nc4GrRGRufzMzFOGdJcBd/vFdwGVDkAdjjBkUqroaaOzlkiXAL9XzAjBGRCYDi4AtqrpVVWPAPf61/RLq7wOyUOBxEVHgJ6q6HJioqrUAqlorIhO6u1FEluH96lGAvHfC8fOIvbWJEtdhV2U1s5INvNFRyJzCDpoao0w48XjerGsj2rQfNIUTihAqKiZSEKK4wKUo7BJ2HUKO4IoggAgIQOcrB1+7jjpPqPL23jYm1u+g7Ohqal5/l/bpM5lYv4MtbilzZk2hZeMbJFIwvroCZ3w10USK+pYYbW1xkrEYqWQCTSXRVMr/R+M9ty8WzJ3Rp+uNGW3Wrl3boKrj+/MMp3yqkmjPep1G924E0i9c7tdzfVEN7Eh7X+Of6+78qX189mHyXekvVtVdfsX+hIi8meuN/j+45QCzwkV644qV7DjrPE4ZW8Q3PvZv3Lf/pyx6+1juP34rv797A5//4/9xxvfXsGHVQyRjUUrGT2PinAVUz6rkvTMrmTelnOryQqqKw5SEXQpCQtgRXAHXfxURHP8HwBGvtvdfSKaUj/7yFW78ry9wzm/+jS8t+BxbvrqCG//rCywZs5h7H/g2z89dxO72BNdefynFf/8dNuyJctuzW3l1fR2NO3YS3bebeFsziY6oX/l7qS/WrFnRp+uNGW1E5N1+PyTRTui4D2W9LP7qz9vTQtdHSro5p72c75e8Vvqqust/rReRB/H+XKkTkcl+K38yUJ/PPBhjTJ+JII47WN9WA0xLez8V2AVEejjfL3mL6YtIiYiUdR4DFwIbgJXAUv+ypcBD+cqDMcYcGcEJRbKmAbIS+IQ/iuc0oMkPgb8EzBaRmSISAa70r+2XfLb0JwIPihcfCQG/UdU/iMhLwL0i8mlgO3BFHvNgjDF9N4AtfRG5GzgbqBKRGuAfgTCAqt4BrAIuAbYAbcDf+p8lROQ64DHABVao6sb+5idvlb6qbgVO6ub8XuC8vjzrQCLFtYVv8aWk8vzeNk4+eTLVyfkUNBSz/521VEZcYqEiYtFEV4xcHBdxBHEE1zkYGuuM1cPBeP3Bz7oPogG8cMqZrHz8l6Te8xU2TTyNE8oLmHdyNRPeM4HE5hYqi1xaEimKXIdwSRGxpNKRSBFLpNCUF4ZLj9/3NZZvjBk8Aog7MJW+ql6V5XMFPt/DZ6vwfhQGTL47co0xJnhEcAYvpj+orNI3xphuDGJH7qCySt8YYzIN7uidQWWVvjHGZBAEJxQe6mzkRSBW2awaU8gTl3+Fi6eVE00qVy6cRsmp51IyfjrNNc1URlxaYiliHQc7cp1wBNd1cB0hEnJw/IlXcHgHbk+dt+kefGMPT575MVZPOIevPfI6C06v5tyZlUxaeAwAYyP4HblCuLyYWEppT6To8DtyO/OVsg5cY4Y/v6WfLQWRtfSNMaYbQa3Us7FK3xhjMokM2JDN4cYqfWOMySCM3JZ+IGL6kZmzeOjdJs78tyuYU1bAOTMqSBz1XsZOGkPj3ijFlUW0xFLEOw6fnBUJOV0LqrkiXfH8vhb8y18+i99t3ccNP/kLLz7xCnOuOYcZso+K+fMRx8Vt3k17SikNObglZcSSSls8STSWJJlM2WQsY4JEHNxQJGsKImvpG2NMJhm5LX2r9I0xJoNg4/SNMWZUGamVfiBi+ptqD/BXE0toveRGFp8xlaLNz7KlWRk3qYzd7UlKJhbT3JEk3u5tYNM5htYNeYutuY7gON2PxpeuzVJ6H62/fumtfOLM6Wx++iH2bnmZyPmfILXuKUJzTyNUVIrbvJtoMkVpyEGKy70F15IpEmnj9DPj+hbnN2aYsnH6xhgzmlh4xxhjRg0RwQkHc3RONlbpG2NMJltwzRhjRpeRWukHoiO3vWkfl973DW56+A3mXbuE+ocf5Lnt+3jvzEr2xZOUTiyhoS1GsiMK4HfiRhARCvzJWd6Ca4LDwYXX+mLpDbez4NFHKRo7kVBRKa9Gy6h94hmilUcTKS4nXvM2sZRSFHFxysZ07ZyVTKZIpQ7dwN46cI0Z/hx/AEhvKYgCUekbY8xgEpGu7VZ7Szk+6yIReUtEtojILd18/mURedVPG0QkKSKV/mfbROQ1/7M1A1E2C+8YY0w3XLf/bWIRcYHbgAuAGuAlEVmpqq93XqOq/w78u3/9B4EbVbUx7THnqGpDvzPjs5a+McZkEgaqpb8I2KKqW1U1BtwDLOnl+quAuwegBD0KRKVfUD6W/y05gycefB4uXMaWh9fx+MbdLJg+hpZEipJJY9jbFiMZi3bd48X1na7JWa5A5g93XwrvFhRx1n/8ifd95BKmn3IOd77wLtuf2czb+zooGjuJeO02kgoF5QU4xWXEkina4kmSiRSqSiqVRFMpNGnxfGOGO2+VzQGp9KuBHWnva/xzh3+nSDFwEXB/2mkFHheRtSKy7MhKcygL7xhjzGG8gR85qMqItS9X1eWHPOhw2s05gA8Cz2eEdhar6i4RmQA8ISJvqurqXDLWE6v0jTEmkx/eyUGDqi7s5fMaYFra+6nArh6uvZKM0I6q7vJf60XkQbxwUb8q/UCEd4wxZrANUHjnJWC2iMwUkQhexb7ysO8SqQDOAh5KO1ciImWdx8CFwIb+lisQLf1jJ5fx1e89RuPWddy9oZ6Otxp5e8te3nPxHJ5NpiitHs+ethiJ2MFx+k7o4Mbo6eP0Rfx4XS9/unX3Z93K//4sZ1/+ZWpX/4gVL+9ixe9eZ+GmRrSmiZKqSRzY7v27iJSGcUrKvQ1U4klSSfXi+jY235jAEAE31P9x+KqaEJHrgMcAF1ihqhtF5Fr/8zv8Sy8HHlfV1rTbJwIP+nVVCPiNqv6hv3kKRKVvjDGDLdvKu7lS1VXAqoxzd2S8/wXwi4xzW4GTBiQTaazSN8aYDCLBnXGbjVX6xhjTjVxn3AaNVfrGGNONkVrpB2L0TuLdt6l//XnKpx7L8j9sYlNLB3veqWF6RZikQkn1eHbvbz9kwTUnHEEcIRJyuzpz+2Pcv36W6ad/gNiPvsznF01l98a1vN0a47nNexgzoYSW7XUAFI4tRMNFtMVTRGPJQzpxrTPXmIAQDi7S2EsKImvpG2NMBkFwQoFoE/eZVfrGGJNJsI5cY4wZTQZqyOZwE4hKf8++dmZ/8nKmHlPJ2kefZX5SaanbRlFTDQCh8dXUN3SQjLUDIK57yIJrEdfBEcGVwzdQcYScNlX5/p0v82jdXdw/+6t8cvp3ad2zg6Z4ik1b9zF+YinNa5pwxVtwLVVQSltr0ovpJ72YvsXzjQkOb8G1oc5FfuS9WCLiisgrIvKI/75SRJ4Qkc3+69h858EYY/pEbOes/rgeeCPt/S3AU6o6G3jKf2+MMcOI4LhO1hREec21iEwFLgV+mnZ6CXCXf3wXcFk+82CMMX0l1tI/Yj8AbgZSaecmqmotgP86obsbRWSZiKwRkTXqpLjny2dy2xUn0lyzicqIS3vTHlLb1hNxhNCk6dQ3t5NKxLo2RXdCERw3PZ5P14Jrjh/b70tHzcdPq0a+9Wk2NHfw4q2P4EaKcAUadjVzQnUFB2pbiDhC4ZgiNFJEWzxJWyxJyh+nb5unGBMsA7VH7nCTt0pfRD4A1Kvq2iO5X1WXq+pCVV1Y7gSiv9kYM0KIcHDXvV5SEOWzNl0MfEhELgEKgXIR+RVQJyKTVbVWRCYD9XnMgzHGHJGgVurZ5K2lr6pfVdWpqjoDb+OA/1PVq/E2EFjqX7aUtE0DjDFmOBCyt/KD+qMwFHGT7wD3isinge3AFUOQB2OM6ZEIRGwZhiOnqs8Az/jHe4Hz+nL/2OOmM/V3/0bRUUcRKizlhPICNJWk/c11lIYcqJjAgdZGknGvI7dzYpbT+Yss3e+GlU36LbP/8Di3jjuB8yeU8ORbexl72QlMee1xmmu38p7qU2nZ5XXkFowtQ8PFtMWbvZ2zUtrViWuTtIwJBhEIBbQln431kBpjTAZh5Mb0rdI3xphMEtyYfTYjM2hljDH94LX0nawpp2eJXCQib4nIFhE5bAUCETlbRJpE5FU/fTPXe49EICr97dEQP7/5Af50y8+ZMG8xx51WTaiwlIb1WxgbdkmWTSR6IIamkt4GKqEwrj9NusCP7QO4DjgIR7J43mk3PcKcsgIu+dFSGmNJjl5wPCeNKaRt7y6OryqhoSNBketQMKaUZKiQjoS3iUoqqRbLNyaABmL0joi4wG3AxcBc4CoRmdvNpc+q6nw/fauP9/aJhXeMMSaDIzJQo3cWAVtUdSuAiNyDtxTN63m+t0eBaOkbY8xgc/3l2HtLQFXncjF+WpbxmGpgR9r7Gv9cptNFZJ2IPCoi8/p4b59YS98YYzJ0LsOQgwZVXdjbo7o5pxnvXwaOUtUWfwWD3wGzc7y3zwLR0t9b18C2tjir3mjg1DNmMvvDp1NcNYU9G2oZX+DSESmjvS3eFdP34vpO1yYqrt8Tnz5WPzOu70j3/4Q71b22mk8+8i22vf9aFowp5Jqzj2baGVOJtzYxtTxCYyxJietQOKaM9kSKlo4E0Viia2P09Ji+xfeNGf4GaEZuDTAt7f1UYFf6BararKot/vEqICwiVbnceySspW+MMRkGcHLWS8BsEZkJ7MRbkuZjh36XTALqVFVFZBFeY3wvsD/bvUfCKn1jjMkgDExHrqomROQ64DHABVao6kYRudb//A7gI8DfiUgCiAJXqqoC3d7b3zxZpW+MMRn6ENPPyg/ZrMo4d0fa8Y+AH+V6b39ZpW+MMRlG8jIMgejIlVCYj587g6QqN593LCXn/jVjph1L4+ZGxheH2deepCPqd+S6Lk44gus6REJeCjveTllwaAduts7bdDd/6wt8p+0k/nb5XzjrirlcfnwVR51/MgCVTgfNiRSlIYeCygqiCaWlPUFbLEkymerquE1ZB64xwWCbqBhjzOjRuZ7+SGSVvjHGdMMqfWOMGSUc20RlaE2bUsmC39xF85mXM58d7CybxaSjGtnR1MGc48exPy2m7zguTiiC68fzXUe8zVTk4EJrR/Kv8ubG3zLlzjbi0RZm/8/NpLY9j3P2pbg/+w2hvdtoSaQ4ttTBqRhHR9JbbC0aS5LqZnIW2AQtY4a1ARy9M9wEotI3xpjBJHStrTPiWKVvjDHdOJItVoPAKn1jjMkggDsy6/xgjNMf29HIpx6r54xbl1J31495dPNe3jd3ArvaE4yZUUHtgQ7irU0Afjw/guMKBX5c3xFvsTWHg+P100kOv+jfWroCcRzEdVkbOY5d99xN69QFFFZUkXhnA9FkiorCEE7FONoTyoGOBInOjdFtsTVjgkXA8fsDe0tBZC19Y4zJIEA4x+0Qg8YqfWOMyTCSwztW6RtjTCYJbvgmG6v0jTEmgzByR+8EImhVv62BlSseoH7xp1j/8xe458/vcs7sKhpjSSpmTGTngXYS7S0AOKHwIZOzIiEHV8B1Di62JiI9Frynf9Fzygr4+jc+yYmXfICvrtzI6/e+yku7WiidOJPoljdJKhSNLcQtG0OrPzErmUiRSqZIpZJoKoUmrRPXmKBwJXsKImvpG2NMBhEIu4FoE/eZVfrGGJNhJId3rNI3xphuBDV8k00g/n5JqpKIRbnu/tdYXdPMOxtqOXlyKbGUUnFMNTX7osSjnTH9CE4oQijs4jr+gmv+5CzIfdOUTFev+y1fTP6JX35mERueXsuLuw7wyMY6xk2bwL5NOwAoqiqC0nEciCU40B4nEU8dNjnLGDP8CQfrjd5STs8SuUhE3hKRLSJySzeff1xE1vvpTyJyUtpn20TkNRF5VUTWDETZrKVvjDGZBmiVTRFxgduAC4Aa4CURWamqr6dd9g5wlqruE5GLgeXAqWmfn6OqDf3OjM8qfWOMyeDF9AfkUYuALaq6FUBE7gGWAF2Vvqr+Ke36F4CpA/LNPQhEeMcYYwZT5zIM2RJQJSJr0tKyjEdVAzvS3tf453ryaeDRtPcKPC4ia7t59hEJREt/wpQKTvzYR/jzw3+kOpZk39Z1VLUtAKDwqFm829BGMhYFwAlHvI3RO8fpuw5h18GVwxdb68uv+ft/uZtPfPdf+cTt22ncupVd7QleWVfLlOljaPz9blyB4qpiUoVltDR64/QT8SSpRKzbTVSMMcOYP7cnBw2qurD3Jx1Gu71Q5By8Sv+MtNOLVXWXiEwAnhCRN1V1dU4560HeWvoiUigiL4rIOhHZKCL/7J+vFJEnRGSz/zo2X3kwxpgj0TlkcwA6cmuAaWnvpwK7Dvs+kROBnwJLVHVv53lV3eW/1gMP4oWL+iWf4Z0O4FxVPQmYD1wkIqcBtwBPqeps4Cn/vTHGDCPezlnZUg5eAmaLyEwRiQBXAisP+SaR6cADwDWquintfImIlHUeAxcCG/pbsryFd1RVgRb/bdhPiteJcbZ//i7gGeAr+cqHMcb01UBNzlLVhIhcBzwGuMAKVd0oItf6n98BfBMYB/zY39sj4YeMJgIP+udCwG9U9Q/9zVNeY/r+cKW1wDHAbar6FxGZqKq1AKpa68equrt3GbAMoHpMKZPymVFjjEnjLcMwMMN3VHUVsCrj3B1px58BPtPNfVuBkzLP91deR++oalJV5+PFsRaJyAl9uHe5qi5U1YWl02axcumJNNdsYlpRmOi+3SReW02RK4SnH0tNYxvJWDviuLihCKFIAY4rRFxv1yxXvF9tkc44XW67ZaVbe9+v2dTSwSPX/4bCivGUhhx2bd7J4tlV7Nu6n4gjlEwoQwvLaOlI0BFLkkqkvE5cW2jNmMARyZ6CaFCGbKrqfrwwzkVAnYhMBvBf6wcjD8YY0xcOkjUFUT5H74wXkTH+cRFwPvAmXifGUv+ypcBD+cqDMcYcCWHktvTzGdOfDNzlx/Ud4F5VfURE/gzcKyKfBrYDV+QxD8YYc0RG6MZZeR29sx44uZvze4Hz+vKsrTv3s+kzf8OYGefz/sibOKkIDc+/QFUkRHLsNJr3bybREUUc11twzXUIhV0iIYew642n7evS2Jm/4qdf/Qm+WBvn1u89y8zTzuKkDSu5c+cmTj/qUnbXt1IacigaP5ZUQRkHYvXEOhLegmvxg5OzbIKWMQER4JZ8NjlVhSLyYX8yVZOINIvIARFpznfmjDFmKMjAjdMfdnJt6X8X+KCqvpHPzBhjzHAx2sM7dVbhG2NGkxFa5+dc6a8Rkf8Ffoe3vAIAqvpAXnJljDFDyLZLhHKgDW/th06Kt15E3iXaW7nrt29yzp3/xHzdT+kDxez802NUF4VoLxlPW/MGNJXECUVwI0WEIi5uyKEg5K2u2bV7FtLtUCtHsv+qP315KW8W38aCny3i5MvnceLWGUQ372ZOVTGvticoD7kUTxhLW0Jp9nfNSnZOzkrrwLXOXGOCYYTW+blV+qr6t/nOiDHGDCcjdbORXEfvTBWRB0WkXkTqROR+Ecnr7i7GGDNUxN8uMVsKolx/zH6ON5N2Ct6uLw/754wxZkQaqTNyc630x6vqz1U14adfAOPzmK9DlI4bR3nI4T+XzKPwsr9n4ux57HqtnsnjitjTlqDtQAepRAxxXZxwpGtiViTkEHaEsHN4LN+Rvg3J+ul7r+bSbz7O5d+8mE/OreDYj3ndG1XxvTTEklRGHAomVNEWT9HUFicRT5JMpkglYgCkLJZvTGAIXuWYLQVRrvluEJGrRcT109XA3qx3GWNMQIlI1hREuVb6nwI+CuwGaoGP+OeMMWbkkYPRgN5SEOU6emc78KE858UYY4YFAQZoD5Vhp9eWvojc7L/+t4j8MDMNThZh9tgwn77lPMY+eRuP7xYWLJjMWwdiVB03jh1NHbQ3NwF0baASCjsURby4fsjfSMXB2zwls8C5/om2Ixpn50urSH7q27T96ju451xDYcV42PYqLYkU4wtCuOMm0xJPsb8tTiKWJBE7dKG1zFdjzPA1WsM7nUsvrMHb9jAzGWPMiOPNyB2Y8I6IXCQib4nIFhG5pZvPxW9IbxGR9SKyINd7j0Sv4R1Vfdg/bFPV+zIyauvgG2NGrIFox/v7idwGXADUAC+JyEpVfT3tsouB2X46FbgdODXHe/ss147cr+Z4zhhjRgB/6ZYsKQeLgC2qulVVY8A9wJKMa5YAv1TPC8AYfyvZXO7ts15b+iJyMXAJUJ0Rwy8HEv39cmOMGZZyn3xVJSJr0t4vV9Xlae+rgR1p72vwWvNkuaY6x3v7LNvonV148fwPcWgM/wBwY3+/PFcHNm0lcd2fWb3o/XzvC6dx80XH8Wh7gvEnTGXdvjbibX5HbqSQUNhbbK044lIUcXEFXOfgv0AR6fHPm95+ub927/U8uWEeV//PK/zD9x8jcsFNVEybQ+srLxBLKaXjiwlVTeJAR5KWjgSppJJKpkilkmgqhSat89aYoBBVJLcBFw2qurC3R3VzTnO8Jpd7+yxbTH8dsE5Efq2q1rI3xowaoqmBeEwNMC3t/VS8xnQu10RyuLfPsg3ZvNc/fMXvVe5Mr4nI+v5+uTHGDE8KmsqesnsJmC0iM0UkAlyJt45ZupXAJ/xRPKcBTapam+O9fZYtvHO9//qB/n6RMcYEivY7koKqJkTkOuAxwAVWqOpGEbnW//wOYBVe3+kWvH1L/ra3e/ubp2zhnVr/sAGIqmpKRI4Fjgce7e+X52pfe4IP/+A5Fm/bz1t/3shZn1vEQymlav6xbK5rIdbqxfSdkLfYmrfgmjc5K9w5OStjo5S+Dsf6up7LM1+cxoQP3cqT25vY8dw7TJldTd0a74e3dEopVEygsS3O/rYYiViSZCJx2CYqxpgAUM21JZ/Do3QVXsWefu6OtGMFPp/rvf2V65DN1UChiFQDT+H9Ev1iIDNijDHDiWgqawqiXCt9UdU24MPAf6vq5cDc/GXLGGOGkkIqkT0FUM6VvoicDnwc+L1/Ltf9dY0xJliUgerIHXZyrfRvwJuB+6DfCXE08HT+snWocSVhNqy6nyJXaNy6jsjrTxFxhMixJ/NGbTOJ9lbEcXELvE3RQ2GXorBL2PHi+WHHK2ZXbP8IFkr68bd/xIsXXELHgUaiyRRrX6xh8UmTqVu3m4gjlE8tJ1k0lqaOBC3tCRLxJKlEjFQ8NtD/OIwxeaeQSmVPAZTr0sp/BP4oImUiUqqqW4Ev5jdrxhgzdIIas88m143R3yMirwAbgNdFZK2IzMtv1owxZgiN0PBOrnH5nwBfUtWnAUTkbOBO4H15ypcxxgwdVRihQ61zrfRLOit8AFV9RkRK8pQnY4wZcqM6vANsFZFviMgMP/0/4J18ZixdyexjqJg+h0tPngRA3SMrmVgQQqfOpa6+lXi0BXFcQpEiwgUu4YLOiVlC2JFDFlxL11OHbnenp55yIb9+YSezz/kgp1UWsefNNXxg3kRqtu6nIuxQNm0iqZJx7IvG6YjGSSZTpOKxrslZNkHLmCAZsGUYhp2+bIw+HnjAT1X4U4WNMWZEGqGVfrb19AuBa4FjgNeAm1Q1PhgZM8aYITOAyzAMN9li+ncBceBZvC295uCN2TfGmBFLGLkx/WyV/lxVfQ+AiPwMeDH/WTrcW3uifOuHSzg1XkXlT1rZ/PAvOaY0THNhFQca15NKxHAjRYSKSr3F1iIuxRFvclbY9bc2Q5BudsPJXIitJ2u+exF/efLbfPza06iOn0z0md2cMqWUH7bGqYqEKJs+keZYioaWDuIdSRKxOKlE7JBYvsX1jQkKhRG68VG2mH5XKKevm6iIyDQReVpE3hCRjSJyvX++UkSeEJHN/uvYI8i3McbkzyhehuEkEWn20wHgxM5jEWnOcm8Crw9gDnAa8HkRmQvcAjylqrPxVuy8pb+FMMaYgTZSV9nMtp6+e6QP9tfir/WPD4jIG3gb/S4BzvYvuwt4BvjKkX6PMcYMvJHbkZvrkM1+EZEZwMnAX4CJnZuz+K8TerhnmYisEZE1rfU7uX7cDt6ZdxlzFp/Ahs2NTJtTxbb9MVr2t6KpJKGCIlx/nH6Rvyl62BVcORjL7yysI17qi5ffdzbnrr6PRdt+z1E3fIVQYSklNS/TEEswqdAlUn0UzbEUe1tixDsS3mJrfkw/ZXF9Y4JnhIZ38r48soiUAvcDN6hqc64rXKrqcmA5gFNc1f99y4wxJlcjeBmGvLb0RSSMV+H/WlUf8E/Xichk//PJQH0+82CMMX2naCKeNfVXLgNbehoU43/2TyKyU0Re9dMl2b4zb5W+eE36nwFvqOp/pn20EljqHy8FHspXHowx5ogoXks/W+q/XAa29DQoptP3VXW+n7Lup5vPlv5i4Brg3Ixfoe8AF4jIZuAC/70xxgwbiqLJZNY0AJbgDWjBf73ssLyo1qrqy/7xAaBzUMwRyVulr6rPqaqo6onpv0KquldVz1PV2f5rY7ZnhUsqeOKDN/J3d7/KP1x4LJtaYkx93yzW1TXTvm83AG6kkEhBiFDYpaww5HXkOtLVmdu5Y1ZPBXay9DXc+1o9F/+2lqc/+33+GJ/CuGMWcODZR4kmlfGTSglPmcG+aILG1hiJuLfYWiqVRFMp7z+QERofNGZEUnLdOauqc8CJn5b18Wp5cxgAABK6SURBVJtyGtjSKWNQTKfrRGS9iKzIZd6T7XNrjDGHybkjt0FVF/Z2gYg8CUzq5qOv9yVHmYNi/NO3A/+C9zP1L8D38BbI7JFV+sYYk0l1QDpqvUfp+T19JiJ1IjJZVWt7G9jSw6AYVLUu7Zo7gUey5WdQxukbY0yw6CF7YfSUBkDWgS29DIrpHAHZ6XK8LW17FYhK/5gp5fxu6z7W/eEZzpuQJJZSJrx/ES9t20d70x4A3IIiwgUhigpDFEVC/iYqDmHHOWxRtc7jvkzQuvmG9/HcXb/i4Xf28c8PbeSY9x7NtsfWAjD26DFo5VTqWmPsbekgljE5yxgTMIM3eqfbgS0iMkVEOkfi9DQoBuC7IvKaiKwHzgFuzPaFFt4xxpjDaGdHbX6/RXUvcF4353cBl/jHz9HDYsCqek1fv9MqfWOMyaQM1JDMYccqfWOMOYwtwzCk3LodnDO+mJa6bUQf/DGVEZfQiWfy+rv7iLU244QihAtLKSgKeWP0wy6FrkOB6+D4C645aWP1oW+bogNsvvYHVB17CtOKwmx85gU+e9bR7Hiuhoqww9jjJpMsn0R9awcHWmMkYkkvnp+0TdGNCSQdnGUYhoK19I0x5jAjt6Vvlb4xxmTqHL0zAlmlb4wxGRRFB2H0zlCwSt8YYzKN4JZ+IDpy6+sOcNk9t1B17Cm8cvtTnFRRQFPFTPbWtpCMRb2O3JIKwgUhSgvDlBWGKAh5O2eFXcHh4O5ZmXKdn3XV9Xfw628t4aPLTqG5ZhMfPG4c6/a3M6UwTOXxR9GUDFHb1E57a5x4R4ykPzmrk3XmGhMgqmg8ljUFkbX0jTHmMIMzOWsoWKVvjDHdGaF/nVulb4wxmVRHbEg2EDH9iCM8VHU+H/jr03n+9QaOe/801tW10lTXgKaShItKiRSXUFAUorQgRGlhiIKQt9iaK4LrHCyoI31baK2TOC7H3//PVH/7JxSPm0LBKw9T15FgVmmEwmPm0tiepHZ/Ox3ROMmOKKm4t9haKm1y1kj9j8iYkUhTqawpiKylb4wxmVTRZDAr9Wys0jfGmAyqSiqeGOps5IVV+sYYk0mxlv5QGj9rEjf9++Os+/FH+Vo0zqwlp/PbzQ207tkOQKiwhILCMIVF4a5N0QtDDmHXH5/P4Zuip2+skm1TdID7friM/5p3GnXTP8Ws089g+90riKWU8fOqcKfPoaa5g9qmKLFogmQs6m+gkhqxy7MaM9JZpW+MMaOEqpIaoQ02q/SNMaYbQR2dk41V+sYYk2mQRu+ISCXwv8AMYBvwUVXd181124ADQBJIqOrCvtyfLhDj9I0xZjB1jt7JlgbALcBTqjobeMp/35NzVHV+Z4V/BPcDAan0650y6jasxr33X6mMuBSdewXPvl5He1MDTihCpKSCwpIwY4rDlBaGKAm7FLj+5Czn0F2zOjt1u9Nbf+70H3ye8pDDQ79+jJs+fAJvPvAGFWGHiQumkxg3g+1NUfY2tRPrSPiduMmuV2NM8KSSqaxpACwB7vKP7wIuy/f9gaj0jTFmUPlDNrMloEpE1qSlZX38pomqWgvgv07oOUc8LiJrM74j1/u7WEzfGGMy5R7Tb8gItxxGRJ4EJnXz0df7kKPFqrpLRCYAT4jIm6q6ug/3d7FK3xhjMigDN3pHVc/v6TMRqRORyapaKyKTgfoenrHLf60XkQeBRcBqIKf70wUivFO7q4Gjz1zCH//f7zhzajk7Cqeze9t+Eu0tuAVFRMoqKSyJUFEcobQwRHHY20AlEhJcObiBypEutgbwH7e9yBdXfIrmmk185CiHvzS2MaskwoSF89gTc9m6p5XogRixaJRELEoqESdlC60ZE0yqpGKJrGkArASW+sdLgYcyLxCREhEp6zwGLgQ25Hp/pkBU+sYYM6gUUqlU1jQAvgNcICKbgQv894jIFBFZ5V8zEXhORNYBLwK/V9U/9HZ/byy8Y4wxGZTBGaevqnuB87o5vwu4xD/eCpzUl/t7Y5W+McZkUkbsulmBCO+oKvfcfBaP1bVw0mfex+83N9C4fTMAkeJyispKKCwOM64kQlkkRFHYW3DN9cfnuxmLraXLZbE1gCvfO5n7Z1/DtFMvZf+KW9nTkeT4YyspPGkx25s72LqnhbaWGIloS9cGKpq0DVSMCSYdsZuo5K3SF5EVIlIvIhvSzlWKyBMistl/HZuv7zfGmCOW+zj9wMlnS/8XwEUZ5/o8ZdgYYwabqpKMJbKmIMpbpe9PHGjMON3fKcfGGDMIRm54Z7A7cg+ZMuzPLuuWP9V4GYAUlA9S9owxhhG9c9aw7chV1eWqulBVF06aPo2Zf/gPKsIu46/5e+7707u01u/ACUUorBhPSXkBVeUFVBSHKQq7FIddClwX16GrMzd9sbX0XbM6ZevPfc/TT3HjP/6Kf/ncqbz4n/9HRdhh+tmziU85gTcbWtm1t4321ljXrlmpVNI6b40JKgVNatYURIPd0u/zlGFjjBlsig7UKprDzmC39Ps8ZdgYYwadgqY0awqivLX0ReRu4Gy8pUdrgH/EmyJ8r4h8GtgOXJGv7zfGmCOlCsnYyAzP5q3SV9WrevioT1OGASa7UX5x4318cPFU1mk12zauI9HeQmHFeArHTqK4vIDxZYWMLY5QFvEmZnUutub2stharhOzABbetIrovjr+pugd/qG+hcXjipl03pnsiDps2NnMgcYoHa0txKMtpBLxrtl8arF9Y4JHgxuzz8aWYTDGmG6krNI3xphRYgQP2bRK3xhjMiiQCmhHbTZW6RtjTCZV68gdSns372RTahJ/f+tNXP3UJvZtXYc4LoVjJ1JeWcy4sUVMKC+gwt81qzjsEnaEsCO4jr97lv+s7iZm5WL3uqe55qbP8sJnvkBSYd4ls3BPvoB1uw+wfsd+Wps7iLc2kYxFSSZi1oFrTICpPzlrJApEpW+MMYPKKn1jjBlNbEauMcaMHoM0IzeXPUZE5DgReTUtNYvIDf5n/yQiO9M+uyTbdwai0o8mlavPPor1k87khdVbaG/aQ0FZJSXjp1M+roipY4uZWF5IRUGI0kiIwrBDyDl8YpakTcZKn5iVyxytG775eW4/8QC/faGGM6uKmXnVEt5lLM9vbWRfXQttTc3EWpu6JmZ1xvQtrm9M8CjeOP1saQBk3WNEVd9S1fmqOh94L9AGPJh2yfc7P1fVVZn3Z7LwjjHGZFIlNTijd5bgLVcD3h4jzwBf6eX684C3VfXdI/3CQLT0jTFmMKkOWkv/kD1GgB73GPFdCdydce46EVnvb1GbdQtaq/SNMaYbOe6cVSUia9LSsszniMiTIrKhm7SkL/kRkQjwIeC+tNO3A7OA+UAt8L1szwlEeGfC+GIW3v0Lzv75y+x54wWcUITSiTOonFjKtImlHFVVTFVxmIrCEIUhh5DLYWP0j3TzlE7faF/Fb89cQZHrcNq174MzruSp1xpYu6WBpr1ttDftIdHeSjIRI+WP0zfGBJTm3JJvUNWFvT9Kz+/pMxHpyx4jFwMvq2pd2rO7jkXkTuCRbBm2lr4xxmQavJ2z+rLHyFVkhHb8H4pOlwMbsn1hIFr6xhgzmJRBW3Ct2z1GRGQK8FNVvcR/XwxcAHwu4/7vish8P8vbuvn8MFbpG2NMJlWSsfxX+qq6l272GFHVXcAlae/bgHHdXHdNX7/TKn1jjMmgCim1ZRiGzpSZXPv0fjY8/iSJ9hYqps9h3FHTmTKtgtkTy6guL6SqOEJZQYiCkDcpq3Nylps+Oct/XOfErD5snMW3P3Y7TfEkyz4yh4l/dwuPbjvAQ6/spL6mmdY9u4i3NpGIRUnFD3biWmeuMcGVtErfGGNGBwVG6HprVukbY0x3rKVvjDGjREohZjtnDZ3Nu5rZ8dP7ie7bTcX0OUw6bi4zjqlkwVFjOW5CKdXlhVQWhykKORS6gogQdry4viPgOtKveD7ArJIwC88/lrk//BGPNpZwx+qtvPNWA/trttPetIdYWzOpuDcxCyyeb0zQWXjHGGNGCUUtvGOMMaOFdeQaY8woY5X+EIq17MdpbWbSSecw5ZjJzDtmHCdOq2Dm2GImlEQoLwhRFBIirkPY8RZXc/1Yfuf4/L5umpLpk6/8L3vHHMN31+zk4T9vYPe2vbTu2U7swD7i0ZauRdYslm9M8Kna6B1jjBk1FBu9Y4wxo4bF9I0xZpSx8I4xxowSXkx/qHORH4Go9IvGVHLaVVewaNY4jp/UucBamJKwS0FICDtycIE1x+ulzezAPZLO23Rn3dNI095naNyxk+i+3cTbmkl0RLs6b60D15iRxVr6xhgzSigwKFuoDAGr9I0xJoOiNnrHGGNGC2/0jlX6Q+b4SaX8/rO9bjjfCz3k5Ui9eM//9O8BxpjgGMEduc5QfKmIXCQib4nIFhG5ZSjyYIwxPels6WdLQTToLX0RcYHb8HZ2rwFeEpGVqvr6YOfFGGN6MlJb+kMR3lkEbFHVrQAicg+wBLBK3xgzLKSwZRgGUjWwI+19DXBq5kUisgxY5r/tKCou3jAIeRssVUDDkdwo8vMBzsqAOeIyDVNWnuGvpzId1d8HNxB77Ce8W5XTpQEzFJV+d9OkDvtJVdXlwHIAEVmjqkfakzvsjLTywMgrk5Vn+MtnmVT1onw8dzgYio7cGmBa2vupwK4hyIcxxow6Q1HpvwTMFpGZIhIBrgRWDkE+jDFm1Bn08I6qJkTkOuAxwAVWqOrGLLctz3/OBtVIKw+MvDJZeYa/kVimvBMN6FhTY4wxfTckk7OMMcYMDav0jTFmFBnWlX5Ql2sQkRUiUi8iG9LOVYrIEyKy2X8dm/bZV/0yviUifzU0ue6ZiEwTkadF5A0R2Sgi1/vnA1kmESkUkRdFZJ1fnn/2zweyPJ1ExBWRV0TkEf990MuzTUReE5FXRWSNfy7QZRoWVHVYJrxO3reBo4EIsA6YO9T5yjHvZwILgA1p574L3OIf3wLc6h/P9ctWAMz0y+wOdRkyyjMZWOAflwGb/HwHskx4c0VK/eMw8BfgtKCWJ61cXwJ+AzwS9P/m/HxuA6oyzgW6TMMhDeeWftdyDaoaAzqXaxj2VHU10Jhxeglwl398F3BZ2vl7VLVDVd8BtuCVfdhQ1VpVfdk/PgC8gTezOpBlUk+L/zbsJyWg5QEQkanApcBP004Htjy9GIllGlTDudLvbrmG6iHKy0CYqKq14FWiwAT/fKDKKSIzgJPxWseBLZMfCnkVqAeeUNVAlwf4AXAzh274FOTygPdD/LiIrPWXZYHgl2nIDef19HNarmEECEw5RaQUuB+4QVWbpeeNh4d9mVQ1CcwXkTHAgyJyQi+XD+vyiMgHgHpVXSsiZ+dySzfnhk150ixW1V0iMgF4QkTe7OXaoJRpyA3nlv5IW66hTkQmA/iv9f75QJRTRMJ4Ff6vVfUB/3SgywSgqvuBZ4CLCG55FgMfEpFteGHQc0XkVwS3PACo6i7/tR54EC9cE+gyDQfDudIfacs1rASW+sdLgYfSzl8pIgUiMhOYDbw4BPnrkXhN+p8Bb6jqf6Z9FMgyich4v4WPiBQB5wNvEtDyqOpXVXWqqs7A+//k/1T1agJaHgARKRGRss5j4EJgAwEu07Ax1D3JvSXgEryRIm8DXx/q/PQh33cDtUAcrwXyaWAc8BSw2X+tTLv+634Z3wIuHur8d1OeM/D+VF4PvOqnS4JaJuBE4BW/PBuAb/rnA1mejLKdzcHRO4EtD96ovXV+2tj5/3+QyzRcki3DYIwxo8hwDu8YY4wZYFbpG2PMKGKVvjHGjCJW6RtjzChilb4xxowiVumbISciSX8lxY3+ypdfEpEj/m9TRL6WdjwjfbVTY0Y7q/TNcBBV1fmqOg+4AG8OwD/243lfy36JMaOTVfpmWFFvyv0y4DrxuCLy7yLykoisF5HPAYjI2SKyWkQeFJHXReQOEXFE5DtAkf+Xw6/9x7oicqf/l8Tj/ixcY0Ylq/TNsKOqW/H+25yAN5u5SVVPAU4BPutPswdvLZabgPcAs4APq+otHPzL4eP+dbOB2/y/JPYDfz14pTFmeLFK3wxXnasmXgh8wl8G+S940/Bn+5+9qN5+C0m8pS/O6OFZ76jqq/7xWmBGfrJszPA3nJdWNqOUiBwNJPFWUBTgC6r6WMY1Z3P40rk9rSnSkXacBCy8Y0Yta+mbYUVExgN3AD9Sb2Gox4C/85d2RkSO9VddBFjkr8LqAH8DPOefj3deb4w5lLX0zXBQ5IdvwkAC+B+gcwnnn+KFY172l3jew8Et8v4MfAcvpr8ab811gOXAehF5GW/lRWOMz1bZNIHkh3f+QVU/MNR5MSZILLxjjDGjiLX0jTFmFLGWvjHGjCJW6RtjzChilb4xxowiVukbY8woYpW+McaMIv8fHGm+9r33PigAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sample_pos_encoding = PositionalEncoding(50, 128)\n",
    "\n",
    "plt.pcolormesh(sample_pos_encoding.pos_encoding.numpy()[0], cmap='RdBu')\n",
    "plt.xlabel('Depth')\n",
    "plt.xlim((0, 512))\n",
    "plt.ylabel('Position')\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scaled_dot_product_attention(query, key, value, mask):\n",
    "    # query 크기 : (batch_size, num_heads, query의 문장 길이, d_model/num_heads)\n",
    "    # key 크기 : (batch_size, num_heads, key의 문장 길이, d_model/num_heads)\n",
    "    # value 크기 : (batch_size, num_heads, value의 문장 길이, d_model/num_heads)\n",
    "    # padding_mask : (batch_size, 1, 1, key의 문장 길이)\n",
    "\n",
    "    # Q와 K의 곱. 어텐션 스코어 행렬.\n",
    "    matmul_qk = tf.matmul(query, key, transpose_b=True)\n",
    "\n",
    "    # 스케일링\n",
    "    # dk의 루트값으로 나눠준다.\n",
    "    depth = tf.cast(tf.shape(key)[-1], tf.float32)\n",
    "    logits = matmul_qk / tf.math.sqrt(depth)\n",
    "\n",
    "    # 마스킹. 어텐션 스코어 행렬의 마스킹 할 위치에 매우 작은 음수값을 넣는다.\n",
    "    # 매우 작은 값이므로 소프트맥스 함수를 지나면 행렬의 해당 위치의 값은 0이 된다.\n",
    "    if mask is not None:\n",
    "        logits += (mask * -1e9)\n",
    "\n",
    "    # 소프트맥스 함수는 마지막 차원인 key의 문장 길이 방향으로 수행된다.\n",
    "    # attention weight : (batch_size, num_heads, query의 문장 길이, key의 문장 길이)\n",
    "    attention_weights = tf.nn.softmax(logits, axis=-1)\n",
    "\n",
    "    # output : (batch_size, num_heads, query의 문장 길이, d_model/num_heads)\n",
    "    output = tf.matmul(attention_weights, value)\n",
    "\n",
    "    return output, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.set_printoptions(suppress=True)\n",
    "temp_k = tf.constant([[10,0,0],\n",
    "                      [0,10,0],\n",
    "                      [0,0,10],\n",
    "                      [0,0,10]], dtype=tf.float32)  # (4, 3)\n",
    "\n",
    "temp_v = tf.constant([[   1,0],\n",
    "                      [  10,0],\n",
    "                      [ 100,5],\n",
    "                      [1000,6]], dtype=tf.float32)  # (4, 2)\n",
    "temp_q = tf.constant([[0, 10, 0]], dtype=tf.float32)  # (1, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([[0.  0.  0.5 0.5]], shape=(1, 4), dtype=float32)\n",
      "tf.Tensor([[550.    5.5]], shape=(1, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "temp_q = tf.constant([[0, 0, 10]], dtype=tf.float32)\n",
    "temp_out, temp_attn = scaled_dot_product_attention(temp_q, temp_k, temp_v, None)\n",
    "print(temp_attn) # 어텐션 분포(어텐션 가중치의 나열)\n",
    "print(temp_out) # 어텐션 값"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[0.  0.  0.5 0.5]\n",
      " [0.  1.  0.  0. ]\n",
      " [0.5 0.5 0.  0. ]], shape=(3, 4), dtype=float32)\n",
      "tf.Tensor(\n",
      "[[550.    5.5]\n",
      " [ 10.    0. ]\n",
      " [  5.5   0. ]], shape=(3, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "temp_q = tf.constant([[0, 0, 10], [0, 10, 0], [10, 10, 0]], dtype=tf.float32)  # (3, 3)\n",
    "temp_out, temp_attn = scaled_dot_product_attention(temp_q, temp_k, temp_v, None)\n",
    "print(temp_attn) # 어텐션 분포(어텐션 가중치의 나열)\n",
    "print(temp_out) # 어텐션 값"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiHeadAttention(tf.keras.layers.Layer):\n",
    "\n",
    "    def __init__(self, d_model, num_heads, name=\"multi_head_attention\"):\n",
    "        super(MultiHeadAttention, self).__init__(name=name)\n",
    "        self.num_heads = num_heads\n",
    "        self.d_model = d_model\n",
    "\n",
    "        assert d_model % self.num_heads == 0\n",
    "\n",
    "        # d_model을 num_heads로 나눈 값.\n",
    "        # 논문 기준 : 64\n",
    "        self.depth = d_model // self.num_heads\n",
    "\n",
    "        # WQ, WK, WV에 해당하는 밀집층 정의\n",
    "        self.query_dense = tf.keras.layers.Dense(units=d_model)\n",
    "        self.key_dense = tf.keras.layers.Dense(units=d_model)\n",
    "        self.value_dense = tf.keras.layers.Dense(units=d_model)\n",
    "\n",
    "        # WO에 해당하는 밀집층 정의\n",
    "        self.dense = tf.keras.layers.Dense(units=d_model)\n",
    "\n",
    "  # num_heads 개수만큼 q, k, v를 split하는 함수\n",
    "    def split_heads(self, inputs, batch_size):\n",
    "        inputs = tf.reshape(\n",
    "            inputs, shape=(batch_size, -1, self.num_heads, self.depth))\n",
    "        return tf.transpose(inputs, perm=[0, 2, 1, 3])\n",
    "\n",
    "    def call(self, inputs):\n",
    "        query, key, value, mask = inputs['query'], inputs['key'], inputs[\n",
    "            'value'], inputs['mask']\n",
    "        batch_size = tf.shape(query)[0]\n",
    "\n",
    "        # 1. WQ, WK, WV에 해당하는 밀집층 지나기\n",
    "        # q : (batch_size, query의 문장 길이, d_model)\n",
    "        # k : (batch_size, key의 문장 길이, d_model)\n",
    "        # v : (batch_size, value의 문장 길이, d_model)\n",
    "        # 참고) 인코더(k, v)-디코더(q) 어텐션에서는 query 길이와 key, value의 길이는 다를 수 있다.\n",
    "        query = self.query_dense(query)\n",
    "        key = self.key_dense(key)\n",
    "        value = self.value_dense(value)\n",
    "\n",
    "        # 2. 헤드 나누기\n",
    "        # q : (batch_size, num_heads, query의 문장 길이, d_model/num_heads)\n",
    "        # k : (batch_size, num_heads, key의 문장 길이, d_model/num_heads)\n",
    "        # v : (batch_size, num_heads, value의 문장 길이, d_model/num_heads)\n",
    "        query = self.split_heads(query, batch_size)\n",
    "        key = self.split_heads(key, batch_size)\n",
    "        value = self.split_heads(value, batch_size)\n",
    "\n",
    "        # 3. 스케일드 닷 프로덕트 어텐션. 앞서 구현한 함수 사용.\n",
    "        # (batch_size, num_heads, query의 문장 길이, d_model/num_heads)\n",
    "        scaled_attention, _ = scaled_dot_product_attention(query, key, value, mask)\n",
    "        # (batch_size, query의 문장 길이, num_heads, d_model/num_heads)\n",
    "        scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])\n",
    "\n",
    "        # 4. 헤드 연결(concatenate)하기\n",
    "        # (batch_size, query의 문장 길이, d_model)\n",
    "        concat_attention = tf.reshape(scaled_attention,\n",
    "                                  (batch_size, -1, self.d_model))\n",
    "\n",
    "        # 5. WO에 해당하는 밀집층 지나기\n",
    "        # (batch_size, query의 문장 길이, d_model)\n",
    "        outputs = self.dense(concat_attention)\n",
    "\n",
    "        return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_padding_mask(x):\n",
    "    mask = tf.cast(tf.math.equal(x, 0), tf.float32)\n",
    "    # (batch_size, 1, 1, key의 문장 길이)\n",
    "    return mask[:, tf.newaxis, tf.newaxis, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([[[[0. 0. 0. 1. 1.]]]], shape=(1, 1, 1, 5), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(create_padding_mask(tf.constant([[1, 21, 777, 0, 0]])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encoder_layer(dff, d_model, num_heads, dropout, name=\"encoder_layer\"):\n",
    "    inputs = tf.keras.Input(shape=(None, d_model), name=\"inputs\")\n",
    "\n",
    "    # 인코더는 패딩 마스크 사용\n",
    "    padding_mask = tf.keras.Input(shape=(1, 1, None), name=\"padding_mask\")\n",
    "\n",
    "    # 멀티-헤드 어텐션 (첫번째 서브층 / 셀프 어텐션)\n",
    "    attention = MultiHeadAttention(\n",
    "        d_model, num_heads, name=\"attention\")({\n",
    "            'query': inputs, 'key': inputs, 'value': inputs, # Q = K = V\n",
    "            'mask': padding_mask # 패딩 마스크 사용\n",
    "        })\n",
    "\n",
    "    # 드롭아웃 + 잔차 연결과 층 정규화\n",
    "    attention = tf.keras.layers.Dropout(rate=dropout)(attention)\n",
    "    attention = tf.keras.layers.LayerNormalization(\n",
    "        epsilon=1e-6)(inputs + attention)\n",
    "\n",
    "    # 포지션 와이즈 피드 포워드 신경망 (두번째 서브층)\n",
    "    outputs = tf.keras.layers.Dense(units=dff, activation='relu')(attention)\n",
    "    outputs = tf.keras.layers.Dense(units=d_model)(outputs)\n",
    "\n",
    "    # 드롭아웃 + 잔차 연결과 층 정규화\n",
    "    outputs = tf.keras.layers.Dropout(rate=dropout)(outputs)\n",
    "    outputs = tf.keras.layers.LayerNormalization(\n",
    "        epsilon=1e-6)(attention + outputs)\n",
    "\n",
    "    return tf.keras.Model(\n",
    "        inputs=[inputs, padding_mask], outputs=outputs, name=name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encoder(vocab_size, num_layers, dff,\n",
    "            d_model, num_heads, dropout,\n",
    "            name=\"encoder\"):\n",
    "    inputs = tf.keras.Input(shape=(None,), name=\"inputs\")\n",
    "\n",
    "    # 인코더는 패딩 마스크 사용\n",
    "    padding_mask = tf.keras.Input(shape=(1, 1, None), name=\"padding_mask\")\n",
    "\n",
    "    # 포지셔널 인코딩 + 드롭아웃\n",
    "    embeddings = tf.keras.layers.Embedding(vocab_size, d_model)(inputs)\n",
    "    embeddings *= tf.math.sqrt(tf.cast(d_model, tf.float32))\n",
    "    embeddings = PositionalEncoding(vocab_size, d_model)(embeddings)\n",
    "    outputs = tf.keras.layers.Dropout(rate=dropout)(embeddings)\n",
    "\n",
    "    # 인코더를 num_layers개 쌓기\n",
    "    for i in range(num_layers):\n",
    "      outputs = encoder_layer(dff=dff, d_model=d_model, num_heads=num_heads,\n",
    "          dropout=dropout, name=\"encoder_layer_{}\".format(i),\n",
    "      )([outputs, padding_mask])\n",
    "\n",
    "    return tf.keras.Model(\n",
    "        inputs=[inputs, padding_mask], outputs=outputs, name=name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_look_ahead_mask(x):\n",
    "    seq_len = tf.shape(x)[1]\n",
    "    look_ahead_mask = 1 - tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0)\n",
    "    padding_mask = create_padding_mask(x) # 패딩 마스크도 포함\n",
    "    return tf.maximum(look_ahead_mask, padding_mask)\n",
    "#     return look_ahead_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[[[0. 1. 1. 1. 1.]\n",
      "   [0. 1. 1. 1. 1.]\n",
      "   [0. 1. 1. 1. 1.]\n",
      "   [0. 1. 1. 0. 1.]\n",
      "   [0. 1. 1. 0. 0.]]]], shape=(1, 1, 5, 5), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(create_look_ahead_mask(tf.constant([[4, 0, 0, 2, 5]])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decoder_layer(dff, d_model, num_heads, dropout, name=\"decoder_layer\"):\n",
    "    inputs = tf.keras.Input(shape=(None, d_model), name=\"inputs\")\n",
    "    enc_outputs = tf.keras.Input(shape=(None, d_model), name=\"encoder_outputs\")\n",
    "\n",
    "    # 디코더는 룩어헤드 마스크(첫번째 서브층)와 패딩 마스크(두번째 서브층) 둘 다 사용.\n",
    "    look_ahead_mask = tf.keras.Input(\n",
    "        shape=(1, None, None), name=\"look_ahead_mask\")\n",
    "    padding_mask = tf.keras.Input(shape=(1, 1, None), name='padding_mask')\n",
    "\n",
    "    # 멀티-헤드 어텐션 (첫번째 서브층 / 마스크드 셀프 어텐션)\n",
    "    attention1 = MultiHeadAttention(\n",
    "        d_model, num_heads, name=\"attention_1\")(inputs={\n",
    "            'query': inputs, 'key': inputs, 'value': inputs, # Q = K = V\n",
    "            'mask': look_ahead_mask # 룩어헤드 마스크\n",
    "        })\n",
    "\n",
    "    # 잔차 연결과 층 정규화\n",
    "    attention1 = tf.keras.layers.LayerNormalization(\n",
    "        epsilon=1e-6)(attention1 + inputs)\n",
    "\n",
    "    # 멀티-헤드 어텐션 (두번째 서브층 / 디코더-인코더 어텐션)\n",
    "    attention2 = MultiHeadAttention(\n",
    "        d_model, num_heads, name=\"attention_2\")(inputs={\n",
    "            'query': attention1, 'key': enc_outputs, 'value': enc_outputs, # Q != K = V\n",
    "            'mask': padding_mask # 패딩 마스크\n",
    "        })\n",
    "\n",
    "    # 드롭아웃 + 잔차 연결과 층 정규화\n",
    "    attention2 = tf.keras.layers.Dropout(rate=dropout)(attention2)\n",
    "    attention2 = tf.keras.layers.LayerNormalization(\n",
    "        epsilon=1e-6)(attention2 + attention1)\n",
    "\n",
    "    # 포지션 와이즈 피드 포워드 신경망 (세번째 서브층)\n",
    "    outputs = tf.keras.layers.Dense(units=dff, activation='relu')(attention2)\n",
    "    outputs = tf.keras.layers.Dense(units=d_model)(outputs)\n",
    "\n",
    "    # 드롭아웃 + 잔차 연결과 층 정규화\n",
    "    outputs = tf.keras.layers.Dropout(rate=dropout)(outputs)\n",
    "    outputs = tf.keras.layers.LayerNormalization(\n",
    "        epsilon=1e-6)(outputs + attention2)\n",
    "\n",
    "    return tf.keras.Model(\n",
    "        inputs=[inputs, enc_outputs, look_ahead_mask, padding_mask],\n",
    "        outputs=outputs,\n",
    "        name=name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decoder(vocab_size, num_layers, dff,\n",
    "            d_model, num_heads, dropout,\n",
    "            name='decoder'):\n",
    "    inputs = tf.keras.Input(shape=(None,), name='inputs')\n",
    "    enc_outputs = tf.keras.Input(shape=(None, d_model), name='encoder_outputs')\n",
    "\n",
    "    # 디코더는 룩어헤드 마스크(첫번째 서브층)와 패딩 마스크(두번째 서브층) 둘 다 사용.\n",
    "    look_ahead_mask = tf.keras.Input(\n",
    "        shape=(1, None, None), name='look_ahead_mask')\n",
    "    padding_mask = tf.keras.Input(shape=(1, 1, None), name='padding_mask')\n",
    "\n",
    "    # 포지셔널 인코딩 + 드롭아웃\n",
    "    embeddings = tf.keras.layers.Embedding(vocab_size, d_model)(inputs)\n",
    "    embeddings *= tf.math.sqrt(tf.cast(d_model, tf.float32))\n",
    "    embeddings = PositionalEncoding(vocab_size, d_model)(embeddings)\n",
    "    outputs = tf.keras.layers.Dropout(rate=dropout)(embeddings)\n",
    "\n",
    "    # 디코더를 num_layers개 쌓기\n",
    "    for i in range(num_layers):\n",
    "        outputs = decoder_layer(dff=dff, d_model=d_model, num_heads=num_heads,\n",
    "            dropout=dropout, name='decoder_layer_{}'.format(i),\n",
    "        )(inputs=[outputs, enc_outputs, look_ahead_mask, padding_mask])\n",
    "\n",
    "    return tf.keras.Model(\n",
    "        inputs=[inputs, enc_outputs, look_ahead_mask, padding_mask],\n",
    "        outputs=outputs,\n",
    "        name=name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transformer(vocab_size, num_layers, dff,\n",
    "                  d_model, num_heads, dropout,\n",
    "                  name=\"transformer\"):\n",
    "\n",
    "    # 인코더의 입력\n",
    "    inputs = tf.keras.Input(shape=(None,), name=\"inputs\")\n",
    "\n",
    "    # 디코더의 입력\n",
    "    dec_inputs = tf.keras.Input(shape=(None,), name=\"dec_inputs\")\n",
    "\n",
    "    # 인코더의 패딩 마스크\n",
    "    enc_padding_mask = tf.keras.layers.Lambda(\n",
    "        create_padding_mask, output_shape=(1, 1, None),\n",
    "        name='enc_padding_mask')(inputs)\n",
    "\n",
    "    # 디코더의 룩어헤드 마스크(첫번째 서브층)\n",
    "    look_ahead_mask = tf.keras.layers.Lambda(\n",
    "        create_look_ahead_mask, output_shape=(1, None, None),\n",
    "        name='look_ahead_mask')(dec_inputs)\n",
    "\n",
    "    # 디코더의 패딩 마스크(두번째 서브층)\n",
    "    dec_padding_mask = tf.keras.layers.Lambda(\n",
    "        create_padding_mask, output_shape=(1, 1, None),\n",
    "        name='dec_padding_mask')(inputs)\n",
    "\n",
    "    # 인코더의 출력은 enc_outputs. 디코더로 전달된다.\n",
    "    enc_outputs = encoder(vocab_size=vocab_size, num_layers=num_layers, dff=dff,\n",
    "        d_model=d_model, num_heads=num_heads, dropout=dropout,\n",
    "    )(inputs=[inputs, enc_padding_mask]) # 인코더의 입력은 입력 문장과 패딩 마스크\n",
    "\n",
    "    # 디코더의 출력은 dec_outputs. 출력층으로 전달된다.\n",
    "    dec_outputs = decoder(vocab_size=vocab_size, num_layers=num_layers, dff=dff,\n",
    "        d_model=d_model, num_heads=num_heads, dropout=dropout,\n",
    "    )(inputs=[dec_inputs, enc_outputs, look_ahead_mask, dec_padding_mask])\n",
    "\n",
    "    # 다음 단어 예측을 위한 출력층\n",
    "    outputs = tf.keras.layers.Dense(units=vocab_size, name=\"outputs\")(dec_outputs)\n",
    "\n",
    "    return tf.keras.Model(inputs=[inputs, dec_inputs], outputs=outputs, name=name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 9000, 128)\n",
      "(1, 9000, 128)\n",
      "('Failed to import pydot. You must `pip install pydot` and install graphviz (https://graphviz.gitlab.io/download/), ', 'for `pydotprint` to work.')\n"
     ]
    }
   ],
   "source": [
    "small_transformer = transformer(\n",
    "    vocab_size = 9000,\n",
    "    num_layers = 4,\n",
    "    dff = 512,\n",
    "    d_model = 128,\n",
    "    num_heads = 4,\n",
    "    dropout = 0.3,\n",
    "    name=\"small_transformer\")\n",
    "\n",
    "tf.keras.utils.plot_model(\n",
    "    small_transformer, to_file='small_transformer.png', show_shapes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_function(y_true, y_pred):\n",
    "    y_true = tf.reshape(y_true, shape=(-1, MAX_LENGTH - 1))\n",
    "\n",
    "    loss = tf.keras.losses.SparseCategoricalCrossentropy(\n",
    "        from_logits=True, reduction='none')(y_true, y_pred)\n",
    "\n",
    "    mask = tf.cast(tf.not_equal(y_true, 0), tf.float32)\n",
    "    loss = tf.multiply(loss, mask)\n",
    "\n",
    "    return tf.reduce_mean(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Train Step')"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZkAAAEGCAYAAAC3lehYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3ycdZ33/9cnk1Nzato0LekxBQqlHJQ2VBBxEU8tLFsVUNhVEL0XccveHnfFdV3x/uneKK4HlIWtu7DgCVF/aIWuyBYBYTmVQwsFKqEFGpqe27RJm0km+dx/XNe00yHJXEnmyjTN+/l4zGOuueb6XvOZK8n1yfdwXV9zd0REROJQVOgARETkyKUkIyIisVGSERGR2CjJiIhIbJRkREQkNsWFDqCQJk2a5I2NjYUOQ0RkVHnyySe3u3t9lG3HdJJpbGxk1apVhQ5DRGRUMbNXo26r5jIREYmNkoyIiMRGSUZERGKjJCMiIrFRkhERkdjEmmTMbJGZrTOzZjO7uo/3zcyuD99fY2bzc5U1s4vMbK2Z9ZpZUx/7nGlm7Wb2+fi+mYiIRBFbkjGzBHADsBiYB1xiZvOyNlsMzAkfVwA3Rij7HPAB4MF+Pvo7wH/l75uIiMhQxVmTWQg0u/t6d+8CbgeWZG2zBLjNA48CtWbWMFBZd3/B3df19YFm9j5gPbA2nq+U251Pt9CeTBXq40VEDitxJplpwMaM1y3huijbRCl7CDOrBL4AfDXHdleY2SozW7Vt27YBv8Bgrd3Uxmd+vpqrf7Umr/sVERmt4kwy1se67BnS+tsmStlsXwW+4+7tA23k7svcvcndm+rrI90VIbJUTxDihu0ded2viMhoFedtZVqAGRmvpwObIm5TGqFstrcAF5rZN4FaoNfMOt39B0OIfUgSRUFu7OzuGamPFBE5rMWZZJ4A5pjZbOB14GLgL7O2WQ5cZWa3EySJNndvNbNtEcoewt3PSi+b2TVA+0gmGIBkqheAzu7ekfxYEZHDVmxJxt1TZnYVcA+QAG5297VmdmX4/k3ACuBcoBnYB1w+UFkAM3s/8H2gHrjbzJ5x9/fG9T0GI5kKajD7VZMREQFivguzu68gSCSZ627KWHZgadSy4fo7gTtzfO41Qwh32NI1mf1dSjIiIqAr/vMqGTaTqSYjIhJQksmjdHOZiIgElGTyKN1cJiIiASWZPMpMMqrViIgoyeRVMqMvpm1/dwEjERE5PCjJ5FFXz8GaTNs+JRkRESWZPEpmXIS5WzUZERElmXzK7JPZrZqMiIiSTD5ldvbv3tdVwEhERA4PSjJ5lEz1UlYcHFLVZERElGTyKtndS11lKSUJY6dqMiIiSjL5lEz1UF6SoK6yjO17k4UOR0Sk4GK9QeZYk0z1UlpcxLjSBDs6VJMREVGSyaNkqpeykgS140rY3q6ajIiImsvyqCvVQ1lxEXVVpexoV01GRERJJo/So8vqq8rY1p4kmC5HRGTsUpLJo2R3L2XFCeqqSulK9dKeTBU6JBGRglKSyaNkqoeykiLqKssA1GQmImOekkweJVO9lCWKmFQdJBl1/ovIWBdrkjGzRWa2zsyazezqPt43M7s+fH+Nmc3PVdbMLjKztWbWa2ZNGevfbWZPmtmz4fM5cX63vgSjy4qoqywFYLtqMiIyxsWWZMwsAdwALAbmAZeY2byszRYDc8LHFcCNEco+B3wAeDBrX9uB8939ZOAy4Ef5/k65JLt7KCtOUK+ajIgIEO91MguBZndfD2BmtwNLgOcztlkC3ObBMKxHzazWzBqAxv7KuvsL4bpDPszdn854uRYoN7Mydx+xM316dNnEsCajPhkRGevibC6bBmzMeN0SrouyTZSyA7kAeLqvBGNmV5jZKjNbtW3btkHscmDuTldPkGRKEkVMqChhW3tn3vYvIjIaxZlkrI912ReO9LdNlLJ9f6jZicA3gE/09b67L3P3Jndvqq+vj7LLSLp7HHcoK0kAMKWmnM1tai4TkbEtzuayFmBGxuvpwKaI25RGKPsGZjYduBO41N1fHkLMQ5aeSyZ9q/+jxpezec/+kQxBROSwE2dN5glgjpnNNrNS4GJgedY2y4FLw1FmpwNt7t4asewhzKwWuBv4ors/nO8vk0t6Vsx0kmkYr5qMiEhsScbdU8BVwD3AC8Ad7r7WzK40syvDzVYA64Fm4IfA3wxUFsDM3m9mLcAZwN1mdk+4r6uAY4Evm9kz4WNyXN8v28EkEzSXHVUzju3tSboypmQWERlrYr0Ls7uvIEgkmetuylh2YGnUsuH6OwmaxLLXfw342jBDHrJkd9BcVnqguSwYxrxlTyczJlYUKiwRkYLSFf95kt1cdtT4cUCQZERExiolmTw5kGRKDvbJALS2KcmIyNilJJMn6eaydJ/MlJogyWxWkhGRMUxJJk+6eg5tLqspL6aiNMFmNZeJyBimJJMnye5DR5eZWXCtjGoyIjKGKcnkSXafDMDU8eN4fbcuyBSRsUtJJk+yr/gHmDFxHC279hUqJBGRglOSyZN0TaY0I8lMn1DB9vYuOjQNs4iMUUoyeZI9ugw4cBFmyy41mYnI2KQkkyfZF2MCzAyTzGs71WQmImOTkkye9JVkZkwIrvrfqCQjImOUkkyedKV6SRQZxYmDh3RiZSkVpQnVZERkzFKSyZNkqueQWgwE18rMnFihEWYiMmYpyeRJMtX7hiQDwQgz1WREZKxSksmTZHfvISPL0mZOrGDjzv0EsxqIiIwtSjJ5kkz1HHK1f9rsSRXs7+7RPcxEZExSksmTZKqX0sQbD+cxk6sAeHlrx0iHJCJScEoyeZJM9fZZkzm2Pkwy29pHOiQRkYJTksmTYHTZG/tk6qvLqC4rVpIRkTEp1iRjZovMbJ2ZNZvZ1X28b2Z2ffj+GjObn6usmV1kZmvNrNfMmrL298Vw+3Vm9t44v1u2oOP/jYfTzDh6cpWSjIiMSbElGTNLADcAi4F5wCVmNi9rs8XAnPBxBXBjhLLPAR8AHsz6vHnAxcCJwCLgX8P9jIiunr6TDMAx9ZXqkxGRMSnOmsxCoNnd17t7F3A7sCRrmyXAbR54FKg1s4aByrr7C+6+ro/PWwLc7u5Jd98ANIf7GRH9DWEGOHZyFZv3dNKuuzGLyBgTZ5KZBmzMeN0SrouyTZSyQ/k8zOwKM1tlZqu2bduWY5fR9TeEGeCYsPN/vZrMRGSMiTPJWB/rsq9I7G+bKGWH8nm4+zJ3b3L3pvr6+hy7jK6/K/4hqMkA/GmLkoyIjC3FMe67BZiR8Xo6sCniNqURyg7l82KTTPUeMmFZpsa6SspLinihdc9IhSMicliIsybzBDDHzGabWSlBp/zyrG2WA5eGo8xOB9rcvTVi2WzLgYvNrMzMZhMMJng8n19oIMnuvocwAySKjOOPquH5TUoyIjK2xFaTcfeUmV0F3AMkgJvdfa2ZXRm+fxOwAjiXoJN+H3D5QGUBzOz9wPeBeuBuM3vG3d8b7vsO4HkgBSx19564vl+2gZrLAOY11LDi2VbcHbO+WvZERI48cTaX4e4rCBJJ5rqbMpYdWBq1bLj+TuDOfsp8Hfj6MEIekp5eJ9Xr/dZkAOY1VPOzx1+jta2TqbXjRjA6EZHC0RX/edCVnhWzn9FlAPOm1gCoyUxExhQlmTxIpoJWuYGay44/Kkwy6vwXkTFESSYPkumazADNZVVlxTTWVagmIyJjipJMHiS700lm4MN58vRaVrfsHomQREQOC0oyeXCguWyAPhmAU2fU0trWSWvb/pEIS0Sk4HImGTM7zsxWmtlz4etTzOwf4w9t9Eg3l/U1aVmmU2fWAvDMa6rNiMjYEKUm80Pgi0A3gLuvIbg4UkIHazID3/R53tQaShNFPL1RSUZExoYoSabC3bOvnNfthDNE7ZMpK05w4rQa1WREZMyIkmS2m9kxhDebNLMLgdZYoxplDo4uy304T50xgTWv76a7pzfusERECi5KklkK/Bsw18xeBz4NXBlrVKNMlCHMaafOrKWzu1c3yxSRMSFKknF3fxfBvcLmuvvbIpYbM6KOLgN4y+yJADy6fkesMYmIHA6iJItfAbh7h7vvDdf9Mr6QRp/BNJdNrinn6PpKHnlZSUZEjnz93iDTzOYCJwLjzewDGW/VAOVxBzaaDKa5DOCMo+v49dOv093TS0mOYc8iIqPZQGe444E/B2qB8zMe84G/jj+00SPZHTSX9TdpWba3HjOJjq4enn29Lc6wREQKrt+ajLv/BviNmZ3h7o+MYEyjzmCaywBOPzrol3nk5R3MnzkhtrhERAotynwyT5vZUoKmswPNZO7+sdiiGmUGm2Tqqso4fko1j7y8g6XvODbO0ERECirKWfFHwFHAe4EHgOnA3gFLjDHJVA+lxUWDmvHyrDmTeHzDTjqSuq5VRI5cUZLMse7+ZaDD3W8FzgNOjjes0aUrx9TLfTnnhMl09fTycPP2mKISESm8KGfG7vB5t5mdBIwHGmOLaBRKpnojjyxLO61xItVlxdz34taYohIRKbwofTLLzGwC8I/AcqAK+HKsUY0yye7B12RKEkW8/bh67ntxK+4+qKY2EZHRIueZ0d3/3d13ufuD7n60u08Gfhdl52a2yMzWmVmzmV3dx/tmZteH768xs/m5yprZRDO718xeCp8nhOtLzOxWM3vWzF4wsy9GOgJ5kEz1RLraP9s75k5m694kazVbpogcoQY8M5rZGWZ2oZlNDl+fYmY/BR7KtWMzSwA3AIuBecAlZjYva7PFwJzwcQVwY4SyVwMr3X0OsDJ8DXARUObuJwMLgE+YWWOuOPNhKM1lAGcfX0+Rwe/Xbo4hKhGRwus3yZjZdcDNwAXA3Wb2FeBe4DGCpJDLQqDZ3de7exdwO7Aka5slwG0eeBSoNbOGHGWXALeGy7cC7wuXHag0s2JgHNAFjEgVIZnqjXwhZqZJVWWcfnQdd61pxd1jiExEpLAGOjOeB5zq7pcA7yGoMbzN3b/n7p0R9j0N2JjxuiVcF2WbgcpOcfdWgPB5crj+l0AHwTQErwHfcved2UGZ2RVmtsrMVm3bti3C18gt2d0z6D6ZtPNOaWD99g6e112ZReQINNCZcX86mbj7LmCdu780iH331ZOd/e96f9tEKZttIdADTAVmA58zs6PfsBP3Ze7e5O5N9fX1OXYZTXIIQ5jTFp/UQKLIuGuNpugRkSPPQGfGY8xsefoBNGa9zqUFmJHxejqwKeI2A5XdEjapET6nxwD/JfA7d+92963Aw0BThDiHbah9MgATK0t56zF13K0mMxE5Ag2UZJYA/5LxyH6dyxPAHDObbWalwMUEQ6AzLQcuDUeZnQ60hU1gA5VdDlwWLl8G/CZcfg04J9xXJXA68GKEOIeta4ijy9LOf9NUXtu5j6c3alpmETmyDHSDzAeGs2N3T5nZVcA9QAK42d3XmtmV4fs3ASuAc4FmYB9w+UBlw11fC9xhZh8nSCwXhetvAG4BniNobrvF3dcM5ztENZzmMoDFJx3FV36zll+s2qgbZorIESXKxZhD5u4rCBJJ5rqbMpadYHrnSGXD9TuAd/axvp2DCWdEDae5DKC6vITzTmlg+TOb+Mfz5lFZFuuPRURkxGjGrDwYzuiytItPm0FHVw93P6sBACJy5FCSyYPhNpcBLJg1gaPrK7njiY25NxYRGSVytsuY2W954/DhNmAV8G8Rr5k5Yrl7XpKMmXHJaTP5+ooXWLupjROnjs9ThCIihRPlzLgeaAd+GD72AFuA48LXY1pXTzhhWcnQ+2TSPnjaDCpKE/zHQxuGvS8RkcNBlCRzqrv/pbv/Nnx8GFjo7kuB+bkKH+kGOyvmQMaPK+GDTTP47epNbN07piuIInKEiHJmrDezmekX4fKk8GVXLFGNIl15TDIAl5/ZSKrX+fEjr+ZlfyIihRTlzPg54CEz+4OZ3Q/8Efi78ILHWwcsOQYcrMkMv7kMYFZdJe86YQo/evRV2jU1s4iMclHmk1lBcNflT4eP4939bnfvcPfvxh3g4S7Z3QMwrCv+sy19x7Hs2tfNbY+8krd9iogUQtQz4wLgROAU4INmdml8IY0u+eyTSXvzjFrecXw9P3xwvWozIjKq5TwzmtmPgG8BbwNOCx8jcuPJ0SDfzWVpn3rXcarNiMioF+X+JU3APNctgvuUbi4byqRlA0nXZpY9uJ6/WjiL8RUled2/iMhIiHJmfA44Ku5ARqs4msvS/n7RXNr2d/P9+wYzjY+IyOEjyplxEvC8md0zyPlkxoS4mssATmio4UNNM7j1kVfYsL0j7/sXEYlblOaya+IOYjRLpvI/uizTZ99zHMtXb+L/rniBZZeqK0xERpecSWa488oc6fJ9MWa2ydXlLH3HsVx3zzrue3EL58ydEsvniIjEod8zo5k9FD7vNbM9GY+9ZrZn5EI8vMXZXJb212cdzZzJVXz512vp0JBmERlF+k0y7v628Lna3WsyHtXuXjNyIR7eDlyMGVNNBoKRa//3Ayfz+u79fPveP8X2OSIi+RbpzGhmCTObamYz04+4AxstDtRkYuqTSWtqnMiHT5/JLQ9v4MlXd8X6WSIi+RLlYsy/Jbi1/73A3eHjrpjjGjXSSaY0Ef/8b19YNJeG8eP4zM+f0Z0ARGRUiHJm/BTB/cpOdPeTw8cpUXZuZovMbJ2ZNZvZ1X28b2Z2ffj+GjObn6usmU00s3vN7KXweULGe6eY2SNmttbMnjWz8ihxDkcy1UOiyCgegSRTXV7Cdy9+My279nHN8rWxf56IyHBFOTNuJJgJc1DMLAHcACwG5gGXmNm8rM0WE9x8cw5wBXBjhLJXAyvdfQ6wMnyNmRUDPwaudPcTgbOB7sHGPVjJ7uHPijkYpzVO5Kp3HMsvn2zht6s3jdjniogMRZTrZNYD95vZ3UAyvdLdv52j3EKg2d3XA5jZ7cAS4PmMbZYAt4W3rHnUzGrNrAFoHKDsEoIEAsFUA/cDXwDeA6xx99VhfDsifLdhy8fUy4P1t++cw0PN2/nCr9Zw/FHVHDelekQ/X0Qkqihnx9cI+mNKgeqMRy7TCGpBaS3huijbDFR2iru3AoTPk8P1xwEe3pngKTP7+76CMrMrzGyVma3atm1bhK8xsK5Ub6zDl/tSkijixg8voKK0mE/86Ena9sdeYRMRGZIBazJhs9WccMrlwbI+1mXfZLO/baKUzVbMwTtF7wNWmtmT7r7ykJ24LwOWATQ1NQ37pp/JVE/sI8v6MqWmnBs/PJ9Llj3KZ37+DD+8tIlEUV+HTUSkcAY8O7p7D8H0y6VD2HcLMCPj9XQguxOhv20GKrslbFIjfN6asa8H3H27u+8DVgDziVkhmsvSTmucyFf+4kTue3Er1yxfi26ULSKHmyhnx1eAh83sy2b22fQjQrkngDlmNjtMUhcD2TfWXA5cGo4yOx1oC5vABiq7HLgsXL4M+E24fA9wiplVhIMA/oxD+39ikSxAc1mmj5w+i0+8/Wh+9Oir3PjAywWLQ0SkL1E6/jeFjyKi9cUA4O4pM7uK4OSfAG5297VmdmX4/k0EtY1zgWaCJq7LByob7vpa4A4z+zhBf9FFYZldZvZtggTlwAp3vztqvEOVTPUUrCaT9oVFc2lt6+Sbv1vHlOpyLlgwvaDxiIikRblB5leHunN3X0GQSDLX3ZSx7MDSqGXD9TuAd/ZT5scEw5hHTLK7N+8Tlg1WUZFx3UWnsKMjyd/9cjWlxUWc/6apBY1JRAQiJBkzqwf+HjgROHBxo7ufE2Nco0Yy1Ut1eZQKYbzKihP88NImPnrLE3z6589gBn9+ihKNiBRWlH/BfwK8CMwGvkrQR/NEjDGNKkFzWeH6ZDJVlBZzy0dPY8HMCXzq9mdYros1RaTAoiSZOnf/D6Db3R9w948Bp8cc16iRTPUWZAhzfyrLirnl8tNYMGsCn7r9af7z4Q2FDklExrAoZ8f0lX6tZnaemZ1KMKRYSF+MefgkGQgSzW0fW8i7T5jCNb99nm/+7kUNbxaRgohydvyamY0HPgd8Hvh34DOxRjWKFHoIc3/KSxLc+OEFXLJwJv96/8t89o7VdIZz34iIjJQoo8vSt/VvA94RbzijT7K78EOY+5MoMv75/ScxdXw5/3Lvn3h5Wzv/9pEFNIwfV+jQRGSMiDKfzHFmttLMngtfn2Jm/xh/aKPD4dYnk83M+Nt3zmHZRxbw8tZ2zv/+w6x6ZWehwxKRMSLK2fGHwBcJ+2bcfQ3BFfhjXqqnl1SvU5o4/JrLsr3nxKP49dIzqSpLcPGyR/nX+5vp7VU/jYjEK0qSqXD3x7PWaVpGoKtnZKZezpc5U6r5zVVv470nHcU3f7eOj9z8GFv3dBY6LBE5gkU5O243s2MI74JsZhcCrbFGNUoku8Mkc5j2yfRl/LgSfnDJqXzjgpN56tXdLPreH1nxrH6cIhKPKGfHpcC/AXPN7HXg08CVsUY1SiRT6SRz+DeXZTIzPnTaTH77t2cytbacv/nJU3zyx0+yda9qNSKSXzmTjLuvd/d3AfXAXHd/G/D+2CMbBbpSo68mk+nYydX8+m/O5AuL5rLyxa28+9sP8ssnW3RNjYjkTeSzo7t3uPve8GWUW/0f8ZKp4LqT0dIn05fiRBGfPPsY/utTZzFnchWf/8VqLrrpEZ57va3QoYnIEWCoZ0dNwcjobS7ryzH1VdzxiTP45gWnsGF7B+f/4CH+4c5n2dnRVejQRGQUG2qSUXsKGTWZUdpclq2oyPjgaTO47/Nnc/lbZ/PzJzbyZ9f9gRv+0ExHUgMKRWTw+j07mtleM9vTx2MvoHvIMzpHl0UxflwJ/3T+PH73qbN4y+w6rrtnHX923R+45eENBxKriEgU/Z4d3b3a3Wv6eFS7e+EnUDkMpJvLCj1pWVzmTKnm3y9r4leffCvHTq7iq799nnO+9QA/few1JRsRieTIPDuOkIPNZaO/T2YgC2ZN4Gd/fTo//vhbmFRdxj/c+SxnfeMPLHvwZdrVjCYiA1CNZBgOdPyP4tFlUZkZb5sziTOPrePh5h3c+EAz/7ziRX5wXzOXntHIR86YxZSa8tw7EpExJdazo5ktMrN1ZtZsZlf38b6Z2fXh+2vMbH6usmY20czuNbOXwucJWfucaWbtZvb5OL8bZI4uO/KTTFo62fzkf53Or5eeyVuPmcQN9zdz5rX3sfSnT/HY+h26zkZEDojt7GhmCeAGYDEwD7jEzOZlbbYYmBM+rgBujFD2amClu88BVoavM30H+K+8f6E+HElDmIfizTNquekjC7j/82dz+ZmN/PFP2/jQskdZ/L0/8tPHXtOINBGJtSazEGgO7xjQBdwOLMnaZglwmwceBWrNrCFH2SXAreHyrcD70jszs/cB64G1cX2pTMnu0X8xZj7MqqvkS+fN47F/eBffuOBkzIx/uPNZTvv6f/O5O1bz6PoduuOzyBgVZ5/MNGBjxusW4C0RtpmWo+wUd28FcPdWM5sMYGaVwBeAdxPM4NknM7uCoNbEzJkzB/eNsozF5rKBjCtN8KHTZvLBphk89doufrGqhbvWtPKrp1qYMXEcF8yfzgXzpzNjYkWhQxWRERJnkunrrgDZ/872t02Ustm+CnzH3dvN+r8hgbsvA5YBNDU1Devf6wNDmBNKMpnMjAWzJrJg1kS+cv6J3LN2M798soXvrXyJ7/73S7x5Ri1/fkoD557cwNRazdIpciSLM8m0ADMyXk8HNkXcpnSAslvMrCGsxTQAW8P1bwEuNLNvArVAr5l1uvsP8vJt+pBM9VBaXMRASW2sG1ea4H2nTuN9p07j9d37Wf7MJu5+dhNfu/sFvnb3CyyYNYHzTm5g8clHaVpokSNQnEnmCWCOmc0GXieYTfMvs7ZZDlxlZrcTJIm2MHlsG6DscuAy4Nrw+TcA7n5Weqdmdg3QHmeCgeCKfzWVRTetdhyfPPsYPnn2MbyyvYO7n23lrjWt/J+7nuf/3PU8J02r4Z1zp/CuE6Zw0rQaJW+RI0BsScbdU2Z2FXAPkABudve1ZnZl+P5NwArgXKAZ2AdcPlDZcNfXAneY2ceB14CL4voOuSRTvWN2ZNlwNU6qZOk7jmXpO47l5W3t/H7tFv77hS1cf99LfG/lSxxVU845J0zmXSdM5oyjJzGuVMdZZDSysXxNQ1NTk69atWrI5T97xzM8tn4nD199Th6jGtt2tCf5w7ptrHxhCw/+aRsdXT2UJopYMGtCeDHoJE6eNp5EkWo5IoViZk+6e1OUbXXF/zB0pXrH/PDlfKurKuPCBdO5cMF0kqkeHt+wk4de2s4fX9rOdfes47p71lFTXsxbj5nEmXMmcdaxk5hVV6GmNZHDlJLMMKi5LF5lxQnOmlPPWXPq+SJBLefhl3fw8Evbeah5O79buxmAo2rKaWqcwGmNE2lqnMDco2pU0xE5TCjJDEOQZFSTGSl1VWX8xZum8hdvmoq788qOfTzUvJ0nNuzkiVd2cteaVgCqy4o5ddYEFjZOoKlxIm+aXqs+HZECUZIZhmR3j5JMgZgZsydVMntSJR85fRYAr+/efyDhrHplF9/6/Z8AKC4yjj+qmlOm1/Km6eN504xa5kyuoljXN4nETklmGJKpXqrLdQgPF9NqxzEtvCYHYPe+Lp56bRdPvrqLNS1t3L1mEz97/DUAxpUkOGlaTZB4ZtRy0tQaGusqKVIzm0he6Qw5DMlUL5PUJ3PYqq0o5Zy5Uzhn7hQAenudV3fuY/XG3axu2c3qjbv58aOv8h8PbQCCxDO3oZoTGmo4oaGGeQ3VHH9UDVVl+jMRGSr99QxDMtWj0WWjSFHRwSa2dG2nu6eXdZv38nzrHp7ftIcXWvdw1+pN/PSx1w6Ua6yrOJB4Tmio4bgpVUyfUKHBBSIRKMkMg674H/1KEkWcNG08J00bf2Cdu7OprZMXwqTzfGvw/Lu1m0lfVlZWXMTR9VUcO7mKOZMPPs+qqzxip+MWGQolmWHo6tEQ5iORmQX9O7XjeNe8KQfWdyRTrNuyl+Yt7TRva+elLXt5+rVd/Hb1wVvyJYqMxroKjp1cxTH1VTSGNadZdRXUV5Xpeh4Zc5RkhkGjy8aWyrJi5s+cwPyZh0zGyr6uFAU3g8sAABFQSURBVOu3ddC8tZ2Xtu4Nn9tZ+cJWUhnz6FSVFTOrriJIPHWVYQKqoLGukomVpUpAckRSkhmGpK74F6CitPgNTW4Q9Pe8vms/G3Z08Mr2Dl7dsY8N2zt47vU2fvfcZnoyElB1WTHTJ1YwfcI4ZkyoYMbEcUzPeNbgAxmt9Js7RO6uK/5lQCWJIhonBTUWjj/0va5ULy279vHKjg5e2b6PV3d00LJrP6/u6OChl7azP5x1NW1CRcmBpDNjQpCMpk+sYFrtOBrGl1NdXjKC30wkOiWZIerq0ayYMnSl4cCBo+ur3vCeu7Ozo4uWXfvZuGsfG3fup2XXPjbu2s+Lm/fy3y9spSucMC+tuqyYhtpyjho/jqnjy2kYP46G2nIawuWpteVUlOrPXUaefuuGSFMvS1zMjLqqMuqqynjTjNo3vN/b62xvT7Jx1z5e391J6+79tLZ1smn3fjbv6eT5TXvY3p58Q7nx40poGF/O5JpyJleXMaWmjCk15UyuLmdyuFxfVabRcZJXSjJDlOxWkpHCKCqyIFHUlLNgVt/bJFM9bGlLsqltP5vbOtnUtp/W3Z20tnWybW8nf9q8l23tyUP6hdLqKkuprw6SzsFEVMakMPHVVZUyqaqMmvJiDVaQnJRkhiiZCtrM1Scjh6Oy4gQz6yqYWVfR7zY9vUGz3JY9nWzbm2TLnk627EmyZW8nW/ck2bq3kxc372Hb3iR95CJKEkZd5cGkk36eVFV6yPpJVWVMrCxVDWmMUpIZogPNZRpdJqNUosiory6jvrpswO16ep0dHUl2tHexo72L7e1Jtrcn2dHRxY72JNvbg+fmre1sb08e+NvIVlNefCDp1FWVBo8wGdVWlDKhooQJFaVMqAyWx5UkVFM6AijJDFGX+mRkjEgUWdBvU12ec1t3p6Or55Dkk37e0XEwQTVvbeexDV3s2tdFf5PzlhUXMaGilNqKEiZWloYJqCRcV8rEyhJqK0qpHVfC+PBRM66EEt1d+7CiJDNEBzv+1VwmkmZmVJUVhxeeVubcPtXTy+793eze18Wufd3s7Ohi974udnak1x1cfmHzHnbvC5b7ar5LqyxNHEg4NRkJqK9HzbjiA9uOH1eiv+cYxJpkzGwR8D0gAfy7u1+b9b6F758L7AM+6u5PDVTWzCYCPwcagVeAD7r7LjN7N3AtUAp0AX/n7vfF9d2S3ek+Gf3XJDJUxYmiA01oUfX2Ons6uw8kpbb9XbTt76ZtXzd7OlPBcsZj4859PBcu7+vqGXDf5SVFhyah8pJDElZ1WTHV5cVUl5dQXV5MVXkxNRmv1cT3RrElGTNLADcA7wZagCfMbLm7P5+x2WJgTvh4C3Aj8JYcZa8GVrr7tWZ2dfj6C8B24Hx332RmJwH3ANPi+n7qkxEpjKIiC5rJKkqZPSl3bSlTd08ve7KSUNv+7gPr9nSmaNt3cH1rWycvbt7Lnv3dtHel+m3aS0sUBTW5A4nowHLwuipcriorprI0SFJVZcVUlhVTVZagMlyuLC0+Yu7yHWdNZiHQ7O7rAczsdmAJkJlklgC3ubsDj5pZrZk1ENRS+iu7BDg7LH8rcD/wBXd/OmO/a4FyMytz9zdeMJAH6SRTmlD1WmS0KEkUHbgGabB6e532rhTtnSn2dqbY29nN3s4Uezq7aU8eum5vxjatbZ38aevB9X0NG+/LuJLEgeRTVR4mpXQSykhK2euqykqoLEtQVVZMRWkxlWUJyosTBZuQL84kMw3YmPG6haC2kmubaTnKTnH3VgB3bzWzyX189gXA03ElGMgYwqyajMiYUFRk1JQHTWhD5e50dvfSnkzRkUwd8hws9xyyvqMrRXvGus17OsPlYF327YcGUlGaOJB0KkqLOWduPX/33rlD/i5RxZlk+kqb2Sm8v22ilO37Q81OBL4BvKef968ArgCYOXNmlF32SRdjishgmRnjShOMK03kHDoeRU+v09EVJqQw+bR3BglpX1eKjq4e9iWznruCZFY5QjddjfNTWoAZGa+nA5siblM6QNktZtYQ1mIagK3pjcxsOnAncKm7v9xXUO6+DFgG0NTUFK3e2geNLhORQkvkoXYVtzj/DX8CmGNms82sFLgYWJ61zXLgUgucDrSFTWEDlV0OXBYuXwb8BsDMaoG7gS+6+8Mxfi8AulIaXSYikktsNRl3T5nZVQSjvBLAze6+1syuDN+/CVhBMHy5mWAI8+UDlQ13fS1wh5l9HHgNuChcfxVwLPBlM/tyuO497n6gppNPGl0mIpJbrI1y7r6CIJFkrrspY9mBpVHLhut3AO/sY/3XgK8NM+TIDo4uU5IREemPzpBDlEz1UFxkFCvJiIj0S2fIIUp296o/RkQkB50lhyiZ6tWty0VEctBZcoiSqR4NXxYRyUFJZoiSqV6NLBMRyUFnySFSn4yISG46Sw5RV0+vmstERHJQkhmioE9Gh09EZCA6Sw5Rslt9MiIiuegsOUTJlJrLRERyUZIZomSqR7eUERHJQWfJIdIQZhGR3HSWHCINYRYRyU1nySHSFf8iIrkpyQxRV0o1GRGRXHSWHCL1yYiI5Kaz5BCkenpJ9bqay0REclCSGYKunnDqZTWXiYgMSGfJIUh2K8mIiEShs+QQJFNBkilVc5mIyIBiTTJmtsjM1plZs5ld3cf7ZmbXh++vMbP5ucqa2UQzu9fMXgqfJ2S898Vw+3Vm9t64vlcy1QOoJiMikktsZ0kzSwA3AIuBecAlZjYva7PFwJzwcQVwY4SyVwMr3X0OsDJ8Tfj+xcCJwCLgX8P95F26JqPRZSIiA4vzLLkQaHb39e7eBdwOLMnaZglwmwceBWrNrCFH2SXAreHyrcD7Mtbf7u5Jd98ANIf7ybuDfTJqLhMRGUicSWYasDHjdUu4Lso2A5Wd4u6tAOHz5EF8HmZ2hZmtMrNV27ZtG9QXSqsqL+a8kxtoGF8+pPIiImNFnEnG+ljnEbeJUnYon4e7L3P3Jndvqq+vz7HLvs2eVMkNfzWfk6aNH1J5EZGxIs4k0wLMyHg9HdgUcZuBym4Jm9QIn7cO4vNERGQExZlkngDmmNlsMysl6JRfnrXNcuDScJTZ6UBb2AQ2UNnlwGXh8mXAbzLWX2xmZWY2m2AwweNxfTkREcmtOK4du3vKzK4C7gESwM3uvtbMrgzfvwlYAZxL0Em/D7h8oLLhrq8F7jCzjwOvAReFZdaa2R3A80AKWOruPXF9PxERyc3cc3V1HLmampp81apVhQ5DRGRUMbMn3b0pyra60ENERGKjJCMiIrFRkhERkdgoyYiISGzGdMe/mW0DXh3GLiYB2/MUTj4prsFRXIOjuAbnSIxrlrtHupp9TCeZ4TKzVVFHWIwkxTU4imtwFNfgjPW41FwmIiKxUZIREZHYKMkMz7JCB9APxTU4imtwFNfgjOm41CcjIiKxUU1GRERioyQjIiLxcXc9BvkAFgHrCO4efXUM+58B/AF4AVgLfCpcfw3wOvBM+Dg3o8wXw3jWAe/NWL8AeDZ873oONpGWAT8P1z8GNA4ivlfCfT4DrArXTQTuBV4KnyeMZGzA8RnH5RlgD/DpQhwz4GaCeY6ey1g3IseHYPqLl8LHZRHiug54EVgD3AnUhusbgf0Zx+2mEY5rRH5uQ4jr5xkxvQI8U4Dj1d/5oeC/Y33+PeT7BHmkPwimHngZOBooBVYD8/L8GQ3A/HC5GvgTMC/8w/t8H9vPC+MoA2aH8SXC9x4HziCYOfS/gMXh+r9J/yEQzNfz80HE9wowKWvdNwkTLnA18I1CxJbxM9oMzCrEMQPeDszn0JNT7MeH4CSzPnyeEC5PyBHXe4DicPkbGXE1Zm6X9f1GIq7Yf25DiSsrln8B/qkAx6u/80PBf8f6eqi5bPAWAs3uvt7du4DbgSX5/AB3b3X3p8LlvQT/sUwboMgS4HZ3T7r7BoL/PhaGM4fWuPsjHvyG3Aa8L6PMreHyL4F3mllfU1hHlbm/W7M+Z6RjeyfwsrsPdDeH2OJy9weBnX18XtzH573Ave6+0913Efw3u2iguNz99+6eCl8+SjCjbL9GKq4BFPR4ZRwHAz4I/GygYGOKq7/zQ8F/x/qiJDN404CNGa9bGDgBDIuZNQKnElRZAa4yszVmdrOZTcgR07Rwua9YD5QJTzJtQF3EsBz4vZk9aWZXhOumeDCrKeHz5ALFBsF/Xpl//IfDMRuJ4zPc382PEfw3mzbbzJ42swfM7KyMzx6puOL+uQ3neJ0FbHH3lzLWjfjxyjo/HJa/Y0oyg9fXf9SxjAM3syrgV8Cn3X0PcCNwDPBmoJWguj5QTAPFOpzvcaa7zwcWA0vN7O0DbDuisYXTdf8F8Itw1eFyzPqTzziGc9y+RDCj7E/CVa3ATHc/Ffgs8FMzqxnBuEbi5zacn+clHPqPzIgfrz7OD/0p6DFTkhm8FoKOt7TpwKZ8f4iZlRD8Av3E3f9/AHff4u497t4L/JCg6W6gmFo4tPkjM9YDZcysGBhPxCYLd98UPm8l6CxeCGwJq9/pJoKthYiNIPE95e5bwhgPi2PGyByfIf1umtllwJ8DfxU2mxA2rewIl58kaMc/bqTiGqGf21CPVzHwAYKO8XS8I3q8+jo/cLj+jg3UYaNHn514xQSdXbM52PF/Yp4/wwjaR7+btb4hY/kzBO2sACdyaMfeeg527D0BnM7Bjr1zw/VLObRj746IsVUC1RnL/0PQJnsdh3Y6fnOkYwu3vx24vNDHjKyO4JE4PgSdsRsIOmQnhMsTc8S1CHgeqM/arj4jjqMJRnpNHMG4Yv+5DSWujGP2QKGOF/2fHw6L37E3/C0M52Q4Vh/AuQQjOl4GvhTD/t9GUAVdQ8YQTuBHBMMN1wDLs/4QvxTGs45whEi4vgl4LnzvBxwcolhO0KTUTDDC5OiIsR0d/sKuJhg++aVwfR2wkmBY48qsP4qRiq0C2AGMz1g34seMoBmlFegm+M/v4yN1fAj6VZrDx+UR4momaGM/ZOgtcEH4810NPAWcP8JxjcjPbbBxhev/E7gya9uRPF79nR8K/jvW10O3lRERkdioT0ZERGKjJCMiIrFRkhERkdgoyYiISGyUZEREJDZKMiJDYGZ1ZvZM+NhsZq9nvC7NUbbJzK4f5Od9zMyeDW+z8pyZLQnXf9TMpg7nu4jESUOYRYbJzK4B2t39Wxnriv3gjSeHu//pwAMEd95tC28nUu/uG8zsfoK7Fa/Kx2eJ5JtqMiJ5Ymb/aWbfNrM/AN8ws4Vm9j/hTRP/x8yOD7c728zuCpevCW8Aeb+ZrTez/93HricDe4F2AHdvDxPMhQQX0/0krEGNM7MF4Q0anzSzezJuM3K/mX03jOM5M1vYx+eI5J2SjEh+HQe8y90/RzAZ2Ns9uGniPwH/3E+ZuQS3UF8IfCW8L1Wm1cAWYIOZ3WJm5wO4+y+BVQT3HHszwQ0uvw9c6O4LCCbd+nrGfird/a0Ec4XcPPyvKpJbcaEDEDnC/MLde8Ll8cCtZjaH4DYg2ckj7W53TwJJM9sKTCHjFuzu3mNmi4DTCObK+Y6ZLXD3a7L2czxwEnBvOM1NguC2KGk/C/f3oJnVmFmtu+8exncVyUlJRiS/OjKW/z/gD+7+/nDej/v7KZPMWO6hj79LDzpPHwceN7N7gVsIZo/MZMBadz+jn8/J7oBVh6zETs1lIvEZT3A3XoCPDnUnZjbVzOZnrHozkJ71cy/BFLwQ3Pyw3szOCMuVmNmJGeU+FK5/G9Dm7m1DjUkkKtVkROLzTYLmss8C9w1jPyXAt8Khyp3ANuDK8L3/BG4ys/0Ec7VfCFxvZuMJ/r6/S3B3YIBdZvY/QA3BnXRFYqchzCJjgIY6S6GouUxERGKjmoyIiMRGNRkREYmNkoyIiMRGSUZERGKjJCMiIrFRkhERkdj8PzIUWFo9n2xoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
    "\n",
    "    def __init__(self, d_model, warmup_steps=4000):\n",
    "        super(CustomSchedule, self).__init__()\n",
    "        self.d_model = d_model\n",
    "        self.d_model = tf.cast(self.d_model, tf.float32)\n",
    "        self.warmup_steps = warmup_steps\n",
    "\n",
    "    def __call__(self, step):\n",
    "        arg1 = tf.math.rsqrt(step)\n",
    "        arg2 = step * (self.warmup_steps**-1.5)\n",
    "\n",
    "        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)\n",
    "sample_learning_rate = CustomSchedule(d_model=128)\n",
    "\n",
    "plt.plot(sample_learning_rate(tf.range(200000, dtype=tf.float32)))\n",
    "plt.ylabel(\"Learning Rate\")\n",
    "plt.xlabel(\"Train Step\")\n",
    "# Text(0.5, 0, 'Train Step')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### tensorflow_datasets 처음 설치 시"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tensorflow_datasets\n",
      "  Downloading tensorflow_datasets-4.1.0-py3-none-any.whl (3.6 MB)\n",
      "Requirement already satisfied: termcolor in c:\\users\\jikim\\anaconda3\\lib\\site-packages (from tensorflow_datasets) (1.1.0)\n",
      "Collecting importlib-resources; python_version < \"3.9\"\n",
      "  Downloading importlib_resources-3.3.0-py2.py3-none-any.whl (26 kB)\n",
      "Requirement already satisfied: six in c:\\users\\jikim\\anaconda3\\lib\\site-packages (from tensorflow_datasets) (1.15.0)\n",
      "Collecting dill\n",
      "  Downloading dill-0.3.3-py2.py3-none-any.whl (81 kB)\n",
      "Requirement already satisfied: numpy in c:\\users\\jikim\\anaconda3\\lib\\site-packages (from tensorflow_datasets) (1.18.5)\n",
      "Collecting promise\n",
      "  Downloading promise-2.3.tar.gz (19 kB)\n",
      "Requirement already satisfied: attrs>=18.1.0 in c:\\users\\jikim\\anaconda3\\lib\\site-packages (from tensorflow_datasets) (19.3.0)\n",
      "Requirement already satisfied: requests>=2.19.0 in c:\\users\\jikim\\anaconda3\\lib\\site-packages (from tensorflow_datasets) (2.24.0)\n",
      "Requirement already satisfied: future in c:\\users\\jikim\\anaconda3\\lib\\site-packages (from tensorflow_datasets) (0.18.2)\n",
      "Requirement already satisfied: protobuf>=3.6.1 in c:\\users\\jikim\\anaconda3\\lib\\site-packages (from tensorflow_datasets) (3.13.0)\n",
      "Collecting tensorflow-metadata\n",
      "  Downloading tensorflow_metadata-0.25.0-py3-none-any.whl (44 kB)\n",
      "Requirement already satisfied: absl-py in c:\\users\\jikim\\anaconda3\\lib\\site-packages (from tensorflow_datasets) (0.10.0)\n",
      "Requirement already satisfied: tqdm in c:\\users\\jikim\\anaconda3\\lib\\site-packages (from tensorflow_datasets) (4.47.0)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\users\\jikim\\anaconda3\\lib\\site-packages (from requests>=2.19.0->tensorflow_datasets) (1.25.9)\n",
      "Requirement already satisfied: idna<3,>=2.5 in c:\\users\\jikim\\anaconda3\\lib\\site-packages (from requests>=2.19.0->tensorflow_datasets) (2.10)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in c:\\users\\jikim\\anaconda3\\lib\\site-packages (from requests>=2.19.0->tensorflow_datasets) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\jikim\\anaconda3\\lib\\site-packages (from requests>=2.19.0->tensorflow_datasets) (2020.6.20)\n",
      "Requirement already satisfied: setuptools in c:\\users\\jikim\\anaconda3\\lib\\site-packages (from protobuf>=3.6.1->tensorflow_datasets) (49.2.0.post20200714)\n",
      "Collecting googleapis-common-protos<2,>=1.52.0\n",
      "  Downloading googleapis_common_protos-1.52.0-py2.py3-none-any.whl (100 kB)\n",
      "Building wheels for collected packages: promise\n",
      "  Building wheel for promise (setup.py): started\n",
      "  Building wheel for promise (setup.py): finished with status 'done'\n",
      "  Created wheel for promise: filename=promise-2.3-py3-none-any.whl size=21499 sha256=7803e8ba1ee705cbf59c0ea915d43f53141f1c755d93e4abbac0ff829ccda3c3\n",
      "  Stored in directory: c:\\users\\jikim\\appdata\\local\\pip\\cache\\wheels\\54\\aa\\01\\724885182f93150035a2a91bce34a12877e8067a97baaf5dc8\n",
      "Successfully built promise\n",
      "Installing collected packages: importlib-resources, dill, promise, googleapis-common-protos, tensorflow-metadata, tensorflow-datasets\n",
      "Successfully installed dill-0.3.3 googleapis-common-protos-1.52.0 importlib-resources-3.3.0 promise-2.3 tensorflow-datasets-4.1.0 tensorflow-metadata-0.25.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: You are using pip version 20.2.3; however, version 20.3 is available.\n",
      "You should consider upgrading via the 'c:\\users\\jikim\\anaconda3\\python.exe -m pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorflow_datasets "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### tensorflow_datasets 업그레이드 시"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip3 install --user --upgrade tensorflow_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import urllib.request\n",
    "import time\n",
    "import tensorflow_datasets as tfds\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.3.1\n"
     ]
    }
   ],
   "source": [
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.1.0\n"
     ]
    }
   ],
   "source": [
    "print(tfds.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Q</th>\n",
       "      <th>A</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12시 땡!</td>\n",
       "      <td>하루가 또 가네요.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1지망 학교 떨어졌어</td>\n",
       "      <td>위로해 드립니다.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3박4일 놀러가고 싶다</td>\n",
       "      <td>여행은 언제나 좋죠.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3박4일 정도 놀러가고 싶다</td>\n",
       "      <td>여행은 언제나 좋죠.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PPL 심하네</td>\n",
       "      <td>눈살이 찌푸려지죠.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Q            A  label\n",
       "0           12시 땡!   하루가 또 가네요.      0\n",
       "1      1지망 학교 떨어졌어    위로해 드립니다.      0\n",
       "2     3박4일 놀러가고 싶다  여행은 언제나 좋죠.      0\n",
       "3  3박4일 정도 놀러가고 싶다  여행은 언제나 좋죠.      0\n",
       "4          PPL 심하네   눈살이 찌푸려지죠.      0"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "urllib.request.urlretrieve(\"https://raw.githubusercontent.com/songys/Chatbot_data/master/ChatbotData%20.csv\", filename=\"ChatBotData.csv\")\n",
    "train_data = pd.read_csv('ChatBotData.csv')\n",
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "챗봇 샘플의 개수 : 11823\n"
     ]
    }
   ],
   "source": [
    "print('챗봇 샘플의 개수 :', len(train_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q        0\n",
      "A        0\n",
      "label    0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(train_data.isnull().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions = []\n",
    "for sentence in train_data['Q']:\n",
    "    # 구두점에 대해서 띄어쓰기\n",
    "    # ex) 12시 땡! -> 12시 땡 !\n",
    "    sentence = re.sub(r\"([?.!,])\", r\" \\1 \", sentence)\n",
    "    sentence = sentence.strip()\n",
    "    questions.append(sentence)\n",
    "answers = []\n",
    "for sentence in train_data['A']:\n",
    "    # 구두점에 대해서 띄어쓰기\n",
    "    # ex) 12시 땡! -> 12시 땡 !\n",
    "    sentence = re.sub(r\"([?.!,])\", r\" \\1 \", sentence)\n",
    "    sentence = sentence.strip()\n",
    "    answers.append(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['12시 땡 !', '1지망 학교 떨어졌어', '3박4일 놀러가고 싶다', '3박4일 정도 놀러가고 싶다', 'PPL 심하네']\n",
      "['하루가 또 가네요 .', '위로해 드립니다 .', '여행은 언제나 좋죠 .', '여행은 언제나 좋죠 .', '눈살이 찌푸려지죠 .']\n"
     ]
    }
   ],
   "source": [
    "print(questions[:5])\n",
    "print(answers[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 서브워드텍스트인코더를 사용하여 질문, 답변 데이터로부터 단어 집합(Vocabulary) 생성\n",
    "tokenizer = tfds.deprecated.text.SubwordTextEncoder.build_from_corpus(\n",
    "    questions + answers, target_vocab_size=2**13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 시작 토큰과 종료 토큰에 대한 정수 부여.\n",
    "START_TOKEN, END_TOKEN = [tokenizer.vocab_size], [tokenizer.vocab_size + 1]\n",
    "\n",
    "# 시작 토큰과 종료 토큰을 고려하여 단어 집합의 크기를 + 2\n",
    "VOCAB_SIZE = tokenizer.vocab_size + 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "시작 토큰 번호 : [8178]\n",
      "종료 토큰 번호 : [8179]\n",
      "단어 집합의 크기 : 8180\n"
     ]
    }
   ],
   "source": [
    "print('시작 토큰 번호 :',START_TOKEN)\n",
    "print('종료 토큰 번호 :',END_TOKEN)\n",
    "print('단어 집합의 크기 :',VOCAB_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "임의의 질문 샘플을 정수 인코딩 : [5766, 611, 3509, 141, 685, 3747, 849]\n"
     ]
    }
   ],
   "source": [
    "# 서브워드텍스트인코더 토크나이저의 .encode()를 사용하여 텍스트 시퀀스를 정수 시퀀스로 변환.\n",
    "print('임의의 질문 샘플을 정수 인코딩 : {}'.format(tokenizer.encode(questions[20])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "정수 인코딩 후의 문장 [5766, 611, 3509, 141, 685, 3747, 849]\n",
      "기존 문장: 가스비 비싼데 감기 걸리겠어\n"
     ]
    }
   ],
   "source": [
    "# 서브워드텍스트인코더 토크나이저의 .encode()와 .decode() 테스트해보기\n",
    "# 임의의 입력 문장을 sample_string에 저장\n",
    "sample_string = questions[20]\n",
    "\n",
    "# encode() : 텍스트 시퀀스 --> 정수 시퀀스\n",
    "tokenized_string = tokenizer.encode(sample_string)\n",
    "print ('정수 인코딩 후의 문장 {}'.format(tokenized_string))\n",
    "\n",
    "# decode() : 정수 시퀀스 --> 텍스트 시퀀스\n",
    "original_string = tokenizer.decode(tokenized_string)\n",
    "print ('기존 문장: {}'.format(original_string))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5766 ----> 가스\n",
      "611 ----> 비 \n",
      "3509 ----> 비싼\n",
      "141 ----> 데 \n",
      "685 ----> 감기 \n",
      "3747 ----> 걸리\n",
      "849 ----> 겠어\n"
     ]
    }
   ],
   "source": [
    "for ts in tokenized_string:\n",
    "    print ('{} ----> {}'.format(ts, tokenizer.decode([ts])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 최대 길이를 40으로 정의\n",
    "MAX_LENGTH = 40\n",
    "\n",
    "# 토큰화 / 정수 인코딩 / 시작 토큰과 종료 토큰 추가 / 패딩\n",
    "def tokenize_and_filter(inputs, outputs):\n",
    "  tokenized_inputs, tokenized_outputs = [], []\n",
    "\n",
    "  for (sentence1, sentence2) in zip(inputs, outputs):\n",
    "    # encode(토큰화 + 정수 인코딩), 시작 토큰과 종료 토큰 추가\n",
    "    sentence1 = START_TOKEN + tokenizer.encode(sentence1) + END_TOKEN\n",
    "    sentence2 = START_TOKEN + tokenizer.encode(sentence2) + END_TOKEN\n",
    "\n",
    "    tokenized_inputs.append(sentence1)\n",
    "    tokenized_outputs.append(sentence2)\n",
    "\n",
    "  # 패딩\n",
    "  tokenized_inputs = tf.keras.preprocessing.sequence.pad_sequences(\n",
    "      tokenized_inputs, maxlen=MAX_LENGTH, padding='post')\n",
    "  tokenized_outputs = tf.keras.preprocessing.sequence.pad_sequences(\n",
    "      tokenized_outputs, maxlen=MAX_LENGTH, padding='post')\n",
    "\n",
    "  return tokenized_inputs, tokenized_outputs\n",
    "questions, answers = tokenize_and_filter(questions, answers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "질문 데이터의 크기(shape) : (11823, 40)\n",
      "답변 데이터의 크기(shape) : (11823, 40)\n"
     ]
    }
   ],
   "source": [
    "print('질문 데이터의 크기(shape) :', questions.shape)\n",
    "print('답변 데이터의 크기(shape) :', answers.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8178 7915 4207 3060   41 8179    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0]\n",
      "[8178 3844   74 7894    1 8179    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0]\n"
     ]
    }
   ],
   "source": [
    "print(questions[0])\n",
    "print(answers[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8178 3844   74 7894    1 8179    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0    0    0    0    0]\n",
      "[[8178 3844   74 7894    1 8179    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0]]\n",
      "[[3844   74 7894    1 8179    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
      "     0    0    0    0    0    0    0    0    0    0    0]]\n"
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 64\n",
    "BUFFER_SIZE = 20000\n",
    "\n",
    "# 디코더의 실제값 시퀀스에서는 시작 토큰을 제거해야 한다.\n",
    "dataset = tf.data.Dataset.from_tensor_slices((\n",
    "    {\n",
    "        'inputs': questions,\n",
    "        'dec_inputs': answers[:, :-1] # 디코더의 입력. 마지막 패딩 토큰이 제거된다.\n",
    "    },\n",
    "    {\n",
    "        'outputs': answers[:, 1:]  # 맨 처음 토큰이 제거된다. 다시 말해 시작 토큰이 제거된다.\n",
    "    },\n",
    "))\n",
    "\n",
    "dataset = dataset.cache()\n",
    "dataset = dataset.shuffle(BUFFER_SIZE)\n",
    "dataset = dataset.batch(BATCH_SIZE)\n",
    "dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)\n",
    "# 임의의 샘플에 대해서 [:, :-1]과 [:, 1:]이 어떤 의미를 가지는지 테스트해본다.\n",
    "print(answers[0]) # 기존 샘플\n",
    "print(answers[:1][:, :-1]) # 마지막 패딩 토큰 제거하면서 길이가 39가 된다.\n",
    "print(answers[:1][:, 1:]) # 맨 처음 토큰이 제거된다. 다시 말해 시작 토큰이 제거된다. 길이는 역시 39가 된다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 8180, 256)\n",
      "(1, 8180, 256)\n"
     ]
    }
   ],
   "source": [
    "D_MODEL = 256\n",
    "NUM_LAYERS = 2\n",
    "NUM_HEADS = 8\n",
    "DFF = 512\n",
    "DROPOUT = 0.1\n",
    "\n",
    "model = transformer(\n",
    "    vocab_size=VOCAB_SIZE,\n",
    "    num_layers=NUM_LAYERS,\n",
    "    dff=DFF,\n",
    "    d_model=D_MODEL,\n",
    "    num_heads=NUM_HEADS,\n",
    "    dropout=DROPOUT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = CustomSchedule(D_MODEL)\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(\n",
    "    learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)\n",
    "\n",
    "def accuracy(y_true, y_pred):\n",
    "  # 레이블의 크기는 (batch_size, MAX_LENGTH - 1)\n",
    "  y_true = tf.reshape(y_true, shape=(-1, MAX_LENGTH - 1))\n",
    "  return tf.keras.metrics.sparse_categorical_accuracy(y_true, y_pred)\n",
    "\n",
    "model.compile(optimizer=optimizer, loss=loss_function, metrics=[accuracy])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "185/185 [==============================] - 268s 1s/step - loss: 1.4557 - accuracy: 0.0232\n",
      "Epoch 2/50\n",
      "185/185 [==============================] - 248s 1s/step - loss: 1.1820 - accuracy: 0.0494\n",
      "Epoch 3/50\n",
      "185/185 [==============================] - 245s 1s/step - loss: 1.0036 - accuracy: 0.0508\n",
      "Epoch 4/50\n",
      "185/185 [==============================] - 246s 1s/step - loss: 0.9281 - accuracy: 0.0543\n",
      "Epoch 5/50\n",
      "185/185 [==============================] - 241s 1s/step - loss: 0.8711 - accuracy: 0.0575\n",
      "Epoch 6/50\n",
      "185/185 [==============================] - 241s 1s/step - loss: 0.8114 - accuracy: 0.0617\n",
      "Epoch 7/50\n",
      "185/185 [==============================] - 245s 1s/step - loss: 0.7457 - accuracy: 0.0674\n",
      "Epoch 8/50\n",
      "185/185 [==============================] - 267s 1s/step - loss: 0.6729 - accuracy: 0.0751\n",
      "Epoch 9/50\n",
      "185/185 [==============================] - 285s 2s/step - loss: 0.5937 - accuracy: 0.0840\n",
      "Epoch 10/50\n",
      "185/185 [==============================] - 275s 1s/step - loss: 0.5113 - accuracy: 0.0930\n",
      "Epoch 11/50\n",
      "185/185 [==============================] - 260s 1s/step - loss: 0.4283 - accuracy: 0.1037\n",
      "Epoch 12/50\n",
      "185/185 [==============================] - 239s 1s/step - loss: 0.3458 - accuracy: 0.1150\n",
      "Epoch 13/50\n",
      "185/185 [==============================] - 245s 1s/step - loss: 0.2720 - accuracy: 0.1257\n",
      "Epoch 14/50\n",
      "185/185 [==============================] - 236s 1s/step - loss: 0.2062 - accuracy: 0.1362\n",
      "Epoch 15/50\n",
      "185/185 [==============================] - 223s 1s/step - loss: 0.1512 - accuracy: 0.1455\n",
      "Epoch 16/50\n",
      "185/185 [==============================] - 222s 1s/step - loss: 0.1106 - accuracy: 0.1530\n",
      "Epoch 17/50\n",
      "185/185 [==============================] - 221s 1s/step - loss: 0.0793 - accuracy: 0.1590\n",
      "Epoch 18/50\n",
      "185/185 [==============================] - 221s 1s/step - loss: 0.0615 - accuracy: 0.1618\n",
      "Epoch 19/50\n",
      "185/185 [==============================] - 222s 1s/step - loss: 0.0508 - accuracy: 0.1635\n",
      "Epoch 20/50\n",
      "185/185 [==============================] - 224s 1s/step - loss: 0.0454 - accuracy: 0.1645\n",
      "Epoch 21/50\n",
      "185/185 [==============================] - 221s 1s/step - loss: 0.0415 - accuracy: 0.1651\n",
      "Epoch 22/50\n",
      "185/185 [==============================] - 225s 1s/step - loss: 0.0400 - accuracy: 0.1654\n",
      "Epoch 23/50\n",
      "185/185 [==============================] - 222s 1s/step - loss: 0.0362 - accuracy: 0.1663\n",
      "Epoch 24/50\n",
      "185/185 [==============================] - 221s 1s/step - loss: 0.0309 - accuracy: 0.1673\n",
      "Epoch 25/50\n",
      "185/185 [==============================] - 221s 1s/step - loss: 0.0271 - accuracy: 0.1683\n",
      "Epoch 26/50\n",
      "185/185 [==============================] - 221s 1s/step - loss: 0.0246 - accuracy: 0.1690\n",
      "Epoch 27/50\n",
      "185/185 [==============================] - 221s 1s/step - loss: 0.0221 - accuracy: 0.1696\n",
      "Epoch 28/50\n",
      "185/185 [==============================] - 218s 1s/step - loss: 0.0200 - accuracy: 0.1702\n",
      "Epoch 29/50\n",
      "185/185 [==============================] - 221s 1s/step - loss: 0.0184 - accuracy: 0.1705\n",
      "Epoch 30/50\n",
      "185/185 [==============================] - 221s 1s/step - loss: 0.0164 - accuracy: 0.1711\n",
      "Epoch 31/50\n",
      "185/185 [==============================] - 221s 1s/step - loss: 0.0151 - accuracy: 0.1714\n",
      "Epoch 32/50\n",
      "185/185 [==============================] - 220s 1s/step - loss: 0.0139 - accuracy: 0.1717\n",
      "Epoch 33/50\n",
      "185/185 [==============================] - 220s 1s/step - loss: 0.0130 - accuracy: 0.1719\n",
      "Epoch 34/50\n",
      "185/185 [==============================] - 220s 1s/step - loss: 0.0118 - accuracy: 0.1723\n",
      "Epoch 35/50\n",
      "185/185 [==============================] - 220s 1s/step - loss: 0.0117 - accuracy: 0.1722\n",
      "Epoch 36/50\n",
      "185/185 [==============================] - 221s 1s/step - loss: 0.0105 - accuracy: 0.1725\n",
      "Epoch 37/50\n",
      "185/185 [==============================] - 224s 1s/step - loss: 0.0102 - accuracy: 0.1725\n",
      "Epoch 38/50\n",
      "185/185 [==============================] - 233s 1s/step - loss: 0.0094 - accuracy: 0.1728\n",
      "Epoch 39/50\n",
      "185/185 [==============================] - 279s 2s/step - loss: 0.0090 - accuracy: 0.1730\n",
      "Epoch 40/50\n",
      "185/185 [==============================] - 268s 1s/step - loss: 0.0088 - accuracy: 0.1731\n",
      "Epoch 41/50\n",
      "185/185 [==============================] - 276s 1s/step - loss: 0.0080 - accuracy: 0.1732\n",
      "Epoch 42/50\n",
      "185/185 [==============================] - 280s 2s/step - loss: 0.0080 - accuracy: 0.1732\n",
      "Epoch 43/50\n",
      "185/185 [==============================] - 270s 1s/step - loss: 0.0074 - accuracy: 0.1733\n",
      "Epoch 44/50\n",
      "185/185 [==============================] - 249s 1s/step - loss: 0.0073 - accuracy: 0.1733\n",
      "Epoch 45/50\n",
      "185/185 [==============================] - 306s 2s/step - loss: 0.0069 - accuracy: 0.1734\n",
      "Epoch 46/50\n",
      "185/185 [==============================] - 296s 2s/step - loss: 0.0067 - accuracy: 0.1735\n",
      "Epoch 47/50\n",
      "185/185 [==============================] - 301s 2s/step - loss: 0.0061 - accuracy: 0.1736\n",
      "Epoch 48/50\n",
      "185/185 [==============================] - 293s 2s/step - loss: 0.0062 - accuracy: 0.1735\n",
      "Epoch 49/50\n",
      "185/185 [==============================] - 273s 1s/step - loss: 0.0058 - accuracy: 0.1737\n",
      "Epoch 50/50\n",
      "185/185 [==============================] - 268s 1s/step - loss: 0.0059 - accuracy: 0.1737\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x214162e6808>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "EPOCHS = 50\n",
    "model.fit(dataset, epochs=EPOCHS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(sentence):\n",
    "  sentence = preprocess_sentence(sentence)\n",
    "\n",
    "  sentence = tf.expand_dims(\n",
    "      START_TOKEN + tokenizer.encode(sentence) + END_TOKEN, axis=0)\n",
    "\n",
    "  output = tf.expand_dims(START_TOKEN, 0)\n",
    "\n",
    "  # 디코더의 예측 시작\n",
    "  for i in range(MAX_LENGTH):\n",
    "    predictions = model(inputs=[sentence, output], training=False)\n",
    "\n",
    "    # 현재(마지막) 시점의 예측 단어를 받아온다.\n",
    "    predictions = predictions[:, -1:, :]\n",
    "    predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n",
    "\n",
    "    # 만약 마지막 시점의 예측 단어가 종료 토큰이라면 예측을 중단\n",
    "    if tf.equal(predicted_id, END_TOKEN[0]):\n",
    "      break\n",
    "\n",
    "    # 마지막 시점의 예측 단어를 출력에 연결한다.\n",
    "    # 이는 for문을 통해서 디코더의 입력으로 사용될 예정이다.\n",
    "    output = tf.concat([output, predicted_id], axis=-1)\n",
    "\n",
    "  return tf.squeeze(output, axis=0)\n",
    "def predict(sentence):\n",
    "  prediction = evaluate(sentence)\n",
    "\n",
    "  predicted_sentence = tokenizer.decode(\n",
    "      [i for i in prediction if i < tokenizer.vocab_size])\n",
    "\n",
    "  print('Input: {}'.format(sentence))\n",
    "  print('Output: {}'.format(predicted_sentence))\n",
    "\n",
    "  return predicted_sentence\n",
    "def preprocess_sentence(sentence):\n",
    "  sentence = re.sub(r\"([?.!,])\", r\" \\1 \", sentence)\n",
    "  sentence = sentence.strip()\n",
    "  return sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: 영화 볼래?\n",
      "Output: 네 말씀해주세요 .\n"
     ]
    }
   ],
   "source": [
    "output = predict(\"영화 볼래?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: 고민이 있어\n",
      "Output: 저는 고민이 없어요 .\n"
     ]
    }
   ],
   "source": [
    "output = predict(\"고민이 있어\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: 너무 화가나\n",
      "Output: 그럴수록 당신이 힘들 거예요 .\n"
     ]
    }
   ],
   "source": [
    "output = predict(\"너무 화가나\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: 카페갈래?\n",
      "Output: 카페 데이트 좋죠 .\n"
     ]
    }
   ],
   "source": [
    "output = predict(\"카페갈래?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: 게임하고싶당\n",
      "Output: 그럴 땐 생각을 덜어봐요 .\n"
     ]
    }
   ],
   "source": [
    "output = predict(\"게임하고싶당\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: 게임하자\n",
      "Output: 게임하세요 !\n"
     ]
    }
   ],
   "source": [
    "output = predict(\"게임하자\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}