Predict_on_GPU_HIRA_UNK.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Predict_on_GPU_HIRA_UNK.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "tZVlCCYugiXc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2ee0e3d3-43c8-45b4-e912-5623077bd399"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YNXjM8H1AIuO",
        "outputId": "77156888-e50a-494c-8964-b37e0b7a8fa5"
      },
      "source": [
        "%%bash \r\n",
        "pwd"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ikEJK6wrGodA"
      },
      "source": [
        "cd /content/drive/My\\ Drive/Transformer-master/\r\n",
        "変更\r\n",
        "cd /content/drive/My\\ Drive/Colab\\ Notebooks/Transformer "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "coo9YpA9cdWA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "38bd9551-b2aa-4a2d-9fb4-0d77d3034d67"
      },
      "source": [
        "cd /content/drive/My\\ Drive/Colab\\ Notebooks/Transformer"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/Colab Notebooks/Transformer\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7QDHMz8U9aJ3"
      },
      "source": [
        "# ライブラリ読み込み"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DQrSy2pW9Xwl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "82ac41ee-c85b-4d9d-c292-d03ea9a63d31"
      },
      "source": [
        "!apt install aptitude\n",
        "!aptitude install mecab libmecab-dev mecab-ipadic-utf8 git make curl xz-utils file -y\n",
        "!pip install mecab-python3==0.6\n",
        "!pip install japanize_matplotlib"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following additional packages will be installed:\n",
            "  aptitude-common libcgi-fast-perl libcgi-pm-perl libclass-accessor-perl\n",
            "  libcwidget3v5 libencode-locale-perl libfcgi-perl libhtml-parser-perl\n",
            "  libhtml-tagset-perl libhttp-date-perl libhttp-message-perl libio-html-perl\n",
            "  libio-string-perl liblwp-mediatypes-perl libparse-debianchangelog-perl\n",
            "  libsigc++-2.0-0v5 libsub-name-perl libtimedate-perl liburi-perl libxapian30\n",
            "Suggested packages:\n",
            "  aptitude-doc-en | aptitude-doc apt-xapian-index debtags tasksel\n",
            "  libcwidget-dev libdata-dump-perl libhtml-template-perl libxml-simple-perl\n",
            "  libwww-perl xapian-tools\n",
            "The following NEW packages will be installed:\n",
            "  aptitude aptitude-common libcgi-fast-perl libcgi-pm-perl\n",
            "  libclass-accessor-perl libcwidget3v5 libencode-locale-perl libfcgi-perl\n",
            "  libhtml-parser-perl libhtml-tagset-perl libhttp-date-perl\n",
            "  libhttp-message-perl libio-html-perl libio-string-perl\n",
            "  liblwp-mediatypes-perl libparse-debianchangelog-perl libsigc++-2.0-0v5\n",
            "  libsub-name-perl libtimedate-perl liburi-perl libxapian30\n",
            "0 upgraded, 21 newly installed, 0 to remove and 13 not upgraded.\n",
            "Need to get 3,877 kB of archives.\n",
            "After this operation, 15.6 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/main amd64 aptitude-common all 0.8.10-6ubuntu1 [1,014 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu bionic/main amd64 libsigc++-2.0-0v5 amd64 2.10.0-2 [10.9 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu bionic/main amd64 libcwidget3v5 amd64 0.5.17-7 [286 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libxapian30 amd64 1.4.5-1ubuntu0.1 [631 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu bionic/main amd64 aptitude amd64 0.8.10-6ubuntu1 [1,269 kB]\n",
            "Get:6 http://archive.ubuntu.com/ubuntu bionic/main amd64 libhtml-tagset-perl all 3.20-3 [12.1 kB]\n",
            "Get:7 http://archive.ubuntu.com/ubuntu bionic/main amd64 liburi-perl all 1.73-1 [77.2 kB]\n",
            "Get:8 http://archive.ubuntu.com/ubuntu bionic/main amd64 libhtml-parser-perl amd64 3.72-3build1 [85.9 kB]\n",
            "Get:9 http://archive.ubuntu.com/ubuntu bionic/main amd64 libcgi-pm-perl all 4.38-1 [185 kB]\n",
            "Get:10 http://archive.ubuntu.com/ubuntu bionic/main amd64 libfcgi-perl amd64 0.78-2build1 [32.8 kB]\n",
            "Get:11 http://archive.ubuntu.com/ubuntu bionic/main amd64 libcgi-fast-perl all 1:2.13-1 [9,940 B]\n",
            "Get:12 http://archive.ubuntu.com/ubuntu bionic/main amd64 libsub-name-perl amd64 0.21-1build1 [11.6 kB]\n",
            "Get:13 http://archive.ubuntu.com/ubuntu bionic/main amd64 libclass-accessor-perl all 0.51-1 [21.2 kB]\n",
            "Get:14 http://archive.ubuntu.com/ubuntu bionic/main amd64 libencode-locale-perl all 1.05-1 [12.3 kB]\n",
            "Get:15 http://archive.ubuntu.com/ubuntu bionic/main amd64 libtimedate-perl all 2.3000-2 [37.5 kB]\n",
            "Get:16 http://archive.ubuntu.com/ubuntu bionic/main amd64 libhttp-date-perl all 6.02-1 [10.4 kB]\n",
            "Get:17 http://archive.ubuntu.com/ubuntu bionic/main amd64 libio-html-perl all 1.001-1 [14.9 kB]\n",
            "Get:18 http://archive.ubuntu.com/ubuntu bionic/main amd64 liblwp-mediatypes-perl all 6.02-1 [21.7 kB]\n",
            "Get:19 http://archive.ubuntu.com/ubuntu bionic/main amd64 libhttp-message-perl all 6.14-1 [72.1 kB]\n",
            "Get:20 http://archive.ubuntu.com/ubuntu bionic/main amd64 libio-string-perl all 1.08-3 [11.1 kB]\n",
            "Get:21 http://archive.ubuntu.com/ubuntu bionic/main amd64 libparse-debianchangelog-perl all 1.2.0-12 [49.5 kB]\n",
            "Fetched 3,877 kB in 1s (3,959 kB/s)\n",
            "Selecting previously unselected package aptitude-common.\n",
            "(Reading database ... 146374 files and directories currently installed.)\n",
            "Preparing to unpack .../00-aptitude-common_0.8.10-6ubuntu1_all.deb ...\n",
            "Unpacking aptitude-common (0.8.10-6ubuntu1) ...\n",
            "Selecting previously unselected package libsigc++-2.0-0v5:amd64.\n",
            "Preparing to unpack .../01-libsigc++-2.0-0v5_2.10.0-2_amd64.deb ...\n",
            "Unpacking libsigc++-2.0-0v5:amd64 (2.10.0-2) ...\n",
            "Selecting previously unselected package libcwidget3v5:amd64.\n",
            "Preparing to unpack .../02-libcwidget3v5_0.5.17-7_amd64.deb ...\n",
            "Unpacking libcwidget3v5:amd64 (0.5.17-7) ...\n",
            "Selecting previously unselected package libxapian30:amd64.\n",
            "Preparing to unpack .../03-libxapian30_1.4.5-1ubuntu0.1_amd64.deb ...\n",
            "Unpacking libxapian30:amd64 (1.4.5-1ubuntu0.1) ...\n",
            "Selecting previously unselected package aptitude.\n",
            "Preparing to unpack .../04-aptitude_0.8.10-6ubuntu1_amd64.deb ...\n",
            "Unpacking aptitude (0.8.10-6ubuntu1) ...\n",
            "Selecting previously unselected package libhtml-tagset-perl.\n",
            "Preparing to unpack .../05-libhtml-tagset-perl_3.20-3_all.deb ...\n",
            "Unpacking libhtml-tagset-perl (3.20-3) ...\n",
            "Selecting previously unselected package liburi-perl.\n",
            "Preparing to unpack .../06-liburi-perl_1.73-1_all.deb ...\n",
            "Unpacking liburi-perl (1.73-1) ...\n",
            "Selecting previously unselected package libhtml-parser-perl.\n",
            "Preparing to unpack .../07-libhtml-parser-perl_3.72-3build1_amd64.deb ...\n",
            "Unpacking libhtml-parser-perl (3.72-3build1) ...\n",
            "Selecting previously unselected package libcgi-pm-perl.\n",
            "Preparing to unpack .../08-libcgi-pm-perl_4.38-1_all.deb ...\n",
            "Unpacking libcgi-pm-perl (4.38-1) ...\n",
            "Selecting previously unselected package libfcgi-perl.\n",
            "Preparing to unpack .../09-libfcgi-perl_0.78-2build1_amd64.deb ...\n",
            "Unpacking libfcgi-perl (0.78-2build1) ...\n",
            "Selecting previously unselected package libcgi-fast-perl.\n",
            "Preparing to unpack .../10-libcgi-fast-perl_1%3a2.13-1_all.deb ...\n",
            "Unpacking libcgi-fast-perl (1:2.13-1) ...\n",
            "Selecting previously unselected package libsub-name-perl.\n",
            "Preparing to unpack .../11-libsub-name-perl_0.21-1build1_amd64.deb ...\n",
            "Unpacking libsub-name-perl (0.21-1build1) ...\n",
            "Selecting previously unselected package libclass-accessor-perl.\n",
            "Preparing to unpack .../12-libclass-accessor-perl_0.51-1_all.deb ...\n",
            "Unpacking libclass-accessor-perl (0.51-1) ...\n",
            "Selecting previously unselected package libencode-locale-perl.\n",
            "Preparing to unpack .../13-libencode-locale-perl_1.05-1_all.deb ...\n",
            "Unpacking libencode-locale-perl (1.05-1) ...\n",
            "Selecting previously unselected package libtimedate-perl.\n",
            "Preparing to unpack .../14-libtimedate-perl_2.3000-2_all.deb ...\n",
            "Unpacking libtimedate-perl (2.3000-2) ...\n",
            "Selecting previously unselected package libhttp-date-perl.\n",
            "Preparing to unpack .../15-libhttp-date-perl_6.02-1_all.deb ...\n",
            "Unpacking libhttp-date-perl (6.02-1) ...\n",
            "Selecting previously unselected package libio-html-perl.\n",
            "Preparing to unpack .../16-libio-html-perl_1.001-1_all.deb ...\n",
            "Unpacking libio-html-perl (1.001-1) ...\n",
            "Selecting previously unselected package liblwp-mediatypes-perl.\n",
            "Preparing to unpack .../17-liblwp-mediatypes-perl_6.02-1_all.deb ...\n",
            "Unpacking liblwp-mediatypes-perl (6.02-1) ...\n",
            "Selecting previously unselected package libhttp-message-perl.\n",
            "Preparing to unpack .../18-libhttp-message-perl_6.14-1_all.deb ...\n",
            "Unpacking libhttp-message-perl (6.14-1) ...\n",
            "Selecting previously unselected package libio-string-perl.\n",
            "Preparing to unpack .../19-libio-string-perl_1.08-3_all.deb ...\n",
            "Unpacking libio-string-perl (1.08-3) ...\n",
            "Selecting previously unselected package libparse-debianchangelog-perl.\n",
            "Preparing to unpack .../20-libparse-debianchangelog-perl_1.2.0-12_all.deb ...\n",
            "Unpacking libparse-debianchangelog-perl (1.2.0-12) ...\n",
            "Setting up libhtml-tagset-perl (3.20-3) ...\n",
            "Setting up libxapian30:amd64 (1.4.5-1ubuntu0.1) ...\n",
            "Setting up libencode-locale-perl (1.05-1) ...\n",
            "Setting up libtimedate-perl (2.3000-2) ...\n",
            "Setting up libio-html-perl (1.001-1) ...\n",
            "Setting up aptitude-common (0.8.10-6ubuntu1) ...\n",
            "Setting up liblwp-mediatypes-perl (6.02-1) ...\n",
            "Setting up liburi-perl (1.73-1) ...\n",
            "Setting up libhtml-parser-perl (3.72-3build1) ...\n",
            "Setting up libcgi-pm-perl (4.38-1) ...\n",
            "Setting up libio-string-perl (1.08-3) ...\n",
            "Setting up libsub-name-perl (0.21-1build1) ...\n",
            "Setting up libfcgi-perl (0.78-2build1) ...\n",
            "Setting up libsigc++-2.0-0v5:amd64 (2.10.0-2) ...\n",
            "Setting up libclass-accessor-perl (0.51-1) ...\n",
            "Setting up libhttp-date-perl (6.02-1) ...\n",
            "Setting up libcgi-fast-perl (1:2.13-1) ...\n",
            "Setting up libparse-debianchangelog-perl (1.2.0-12) ...\n",
            "Setting up libhttp-message-perl (6.14-1) ...\n",
            "Setting up libcwidget3v5:amd64 (0.5.17-7) ...\n",
            "Setting up aptitude (0.8.10-6ubuntu1) ...\n",
            "update-alternatives: using /usr/bin/aptitude-curses to provide /usr/bin/aptitude (aptitude) in auto mode\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1.3) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/python3.6/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
            "\n",
            "git is already installed at the requested version (1:2.17.1-1ubuntu0.7)\n",
            "make is already installed at the requested version (4.1-9.1ubuntu1)\n",
            "curl is already installed at the requested version (7.58.0-2ubuntu3.12)\n",
            "xz-utils is already installed at the requested version (5.2.2-1.3)\n",
            "git is already installed at the requested version (1:2.17.1-1ubuntu0.7)\n",
            "make is already installed at the requested version (4.1-9.1ubuntu1)\n",
            "curl is already installed at the requested version (7.58.0-2ubuntu3.12)\n",
            "xz-utils is already installed at the requested version (5.2.2-1.3)\n",
            "The following NEW packages will be installed:\n",
            "  file libmagic-mgc{a} libmagic1{a} libmecab-dev libmecab2{a} mecab mecab-ipadic{a} mecab-ipadic-utf8 mecab-jumandic{a} mecab-jumandic-utf8{a} mecab-utils{a} \n",
            "0 packages upgraded, 11 newly installed, 0 to remove and 13 not upgraded.\n",
            "Need to get 29.3 MB of archives. After unpacking 282 MB will be used.\n",
            "Get: 1 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libmagic-mgc amd64 1:5.32-2ubuntu0.4 [184 kB]\n",
            "Get: 2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libmagic1 amd64 1:5.32-2ubuntu0.4 [68.6 kB]\n",
            "Get: 3 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 file amd64 1:5.32-2ubuntu0.4 [22.1 kB]\n",
            "Get: 4 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libmecab2 amd64 0.996-5 [257 kB]\n",
            "Get: 5 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libmecab-dev amd64 0.996-5 [308 kB]\n",
            "Get: 6 http://archive.ubuntu.com/ubuntu bionic/universe amd64 mecab-utils amd64 0.996-5 [4,856 B]\n",
            "Get: 7 http://archive.ubuntu.com/ubuntu bionic/universe amd64 mecab-jumandic-utf8 all 7.0-20130310-4 [16.2 MB]\n",
            "Get: 8 http://archive.ubuntu.com/ubuntu bionic/universe amd64 mecab-jumandic all 7.0-20130310-4 [2,212 B]\n",
            "Get: 9 http://archive.ubuntu.com/ubuntu bionic/universe amd64 mecab-ipadic all 2.7.0-20070801+main-1 [12.1 MB]\n",
            "Get: 10 http://archive.ubuntu.com/ubuntu bionic/universe amd64 mecab amd64 0.996-5 [132 kB]\n",
            "Get: 11 http://archive.ubuntu.com/ubuntu bionic/universe amd64 mecab-ipadic-utf8 all 2.7.0-20070801+main-1 [3,522 B]\n",
            "Fetched 29.3 MB in 2s (18.7 MB/s)\n",
            "Selecting previously unselected package libmagic-mgc.\n",
            "(Reading database ... 146833 files and directories currently installed.)\n",
            "Preparing to unpack .../00-libmagic-mgc_1%3a5.32-2ubuntu0.4_amd64.deb ...\n",
            "Unpacking libmagic-mgc (1:5.32-2ubuntu0.4) ...\n",
            "Selecting previously unselected package libmagic1:amd64.\n",
            "Preparing to unpack .../01-libmagic1_1%3a5.32-2ubuntu0.4_amd64.deb ...\n",
            "Unpacking libmagic1:amd64 (1:5.32-2ubuntu0.4) ...\n",
            "Selecting previously unselected package file.\n",
            "Preparing to unpack .../02-file_1%3a5.32-2ubuntu0.4_amd64.deb ...\n",
            "Unpacking file (1:5.32-2ubuntu0.4) ...\n",
            "Selecting previously unselected package libmecab2:amd64.\n",
            "Preparing to unpack .../03-libmecab2_0.996-5_amd64.deb ...\n",
            "Unpacking libmecab2:amd64 (0.996-5) ...\n",
            "Selecting previously unselected package libmecab-dev.\n",
            "Preparing to unpack .../04-libmecab-dev_0.996-5_amd64.deb ...\n",
            "Unpacking libmecab-dev (0.996-5) ...\n",
            "Selecting previously unselected package mecab-utils.\n",
            "Preparing to unpack .../05-mecab-utils_0.996-5_amd64.deb ...\n",
            "Unpacking mecab-utils (0.996-5) ...\n",
            "Selecting previously unselected package mecab-jumandic-utf8.\n",
            "Preparing to unpack .../06-mecab-jumandic-utf8_7.0-20130310-4_all.deb ...\n",
            "Unpacking mecab-jumandic-utf8 (7.0-20130310-4) ...\n",
            "Selecting previously unselected package mecab-jumandic.\n",
            "Preparing to unpack .../07-mecab-jumandic_7.0-20130310-4_all.deb ...\n",
            "Unpacking mecab-jumandic (7.0-20130310-4) ...\n",
            "Selecting previously unselected package mecab-ipadic.\n",
            "Preparing to unpack .../08-mecab-ipadic_2.7.0-20070801+main-1_all.deb ...\n",
            "Unpacking mecab-ipadic (2.7.0-20070801+main-1) ...\n",
            "Selecting previously unselected package mecab.\n",
            "Preparing to unpack .../09-mecab_0.996-5_amd64.deb ...\n",
            "Unpacking mecab (0.996-5) ...\n",
            "Selecting previously unselected package mecab-ipadic-utf8.\n",
            "Preparing to unpack .../10-mecab-ipadic-utf8_2.7.0-20070801+main-1_all.deb ...\n",
            "Unpacking mecab-ipadic-utf8 (2.7.0-20070801+main-1) ...\n",
            "Setting up libmecab2:amd64 (0.996-5) ...\n",
            "Setting up libmagic-mgc (1:5.32-2ubuntu0.4) ...\n",
            "Setting up libmagic1:amd64 (1:5.32-2ubuntu0.4) ...\n",
            "Setting up mecab-utils (0.996-5) ...\n",
            "Setting up mecab-ipadic (2.7.0-20070801+main-1) ...\n",
            "Compiling IPA dictionary for Mecab.  This takes long time...\n",
            "reading /usr/share/mecab/dic/ipadic/unk.def ... 40\n",
            "emitting double-array: 100% |###########################################| \n",
            "/usr/share/mecab/dic/ipadic/model.def is not found. skipped.\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.verbal.csv ... 12146\n",
            "reading /usr/share/mecab/dic/ipadic/Suffix.csv ... 1393\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.nai.csv ... 42\n",
            "reading /usr/share/mecab/dic/ipadic/Adverb.csv ... 3032\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.adverbal.csv ... 795\n",
            "reading /usr/share/mecab/dic/ipadic/Symbol.csv ... 208\n",
            "reading /usr/share/mecab/dic/ipadic/Adnominal.csv ... 135\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.others.csv ... 151\n",
            "reading /usr/share/mecab/dic/ipadic/Postp-col.csv ... 91\n",
            "reading /usr/share/mecab/dic/ipadic/Prefix.csv ... 221\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.adjv.csv ... 3328\n",
            "reading /usr/share/mecab/dic/ipadic/Interjection.csv ... 252\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.place.csv ... 72999\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.number.csv ... 42\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.org.csv ... 16668\n",
            "reading /usr/share/mecab/dic/ipadic/Filler.csv ... 19\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.csv ... 60477\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.name.csv ... 34202\n",
            "reading /usr/share/mecab/dic/ipadic/Others.csv ... 2\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.demonst.csv ... 120\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.proper.csv ... 27327\n",
            "reading /usr/share/mecab/dic/ipadic/Verb.csv ... 130750\n",
            "reading /usr/share/mecab/dic/ipadic/Auxil.csv ... 199\n",
            "reading /usr/share/mecab/dic/ipadic/Adj.csv ... 27210\n",
            "reading /usr/share/mecab/dic/ipadic/Postp.csv ... 146\n",
            "reading /usr/share/mecab/dic/ipadic/Conjunction.csv ... 171\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading /usr/share/mecab/dic/ipadic/matrix.def ... 1316x1316\n",
            "emitting matrix      : 100% |###########################################| \n",
            "\n",
            "done!\n",
            "update-alternatives: using /var/lib/mecab/dic/ipadic to provide /var/lib/mecab/dic/debian (mecab-dictionary) in auto mode\n",
            "Setting up libmecab-dev (0.996-5) ...\n",
            "Setting up file (1:5.32-2ubuntu0.4) ...\n",
            "Setting up mecab-jumandic-utf8 (7.0-20130310-4) ...\n",
            "Compiling Juman dictionary for Mecab.\n",
            "reading /usr/share/mecab/dic/juman/unk.def ... 37\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading /usr/share/mecab/dic/juman/Assert.csv ... 34\n",
            "reading /usr/share/mecab/dic/juman/ContentW.csv ... 551145\n",
            "reading /usr/share/mecab/dic/juman/Noun.koyuu.csv ... 7964\n",
            "reading /usr/share/mecab/dic/juman/Suffix.csv ... 2128\n",
            "reading /usr/share/mecab/dic/juman/Noun.keishiki.csv ... 8\n",
            "reading /usr/share/mecab/dic/juman/Wikipedia.csv ... 167709\n",
            "reading /usr/share/mecab/dic/juman/Prefix.csv ... 90\n",
            "reading /usr/share/mecab/dic/juman/Noun.hukusi.csv ... 81\n",
            "reading /usr/share/mecab/dic/juman/Emoticon.csv ... 972\n",
            "reading /usr/share/mecab/dic/juman/Demonstrative.csv ... 97\n",
            "reading /usr/share/mecab/dic/juman/Special.csv ... 158\n",
            "reading /usr/share/mecab/dic/juman/Auto.csv ... 18931\n",
            "reading /usr/share/mecab/dic/juman/Postp.csv ... 108\n",
            "reading /usr/share/mecab/dic/juman/Noun.suusi.csv ... 49\n",
            "reading /usr/share/mecab/dic/juman/Rengo.csv ... 1118\n",
            "reading /usr/share/mecab/dic/juman/AuxV.csv ... 593\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading /usr/share/mecab/dic/juman/matrix.def ... 1876x1876\n",
            "emitting matrix      : 100% |###########################################| \n",
            "\n",
            "done!\n",
            "Setting up mecab-ipadic-utf8 (2.7.0-20070801+main-1) ...\n",
            "Compiling IPA dictionary for Mecab.  This takes long time...\n",
            "reading /usr/share/mecab/dic/ipadic/unk.def ... 40\n",
            "emitting double-array: 100% |###########################################| \n",
            "/usr/share/mecab/dic/ipadic/model.def is not found. skipped.\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.verbal.csv ... 12146\n",
            "reading /usr/share/mecab/dic/ipadic/Suffix.csv ... 1393\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.nai.csv ... 42\n",
            "reading /usr/share/mecab/dic/ipadic/Adverb.csv ... 3032\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.adverbal.csv ... 795\n",
            "reading /usr/share/mecab/dic/ipadic/Symbol.csv ... 208\n",
            "reading /usr/share/mecab/dic/ipadic/Adnominal.csv ... 135\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.others.csv ... 151\n",
            "reading /usr/share/mecab/dic/ipadic/Postp-col.csv ... 91\n",
            "reading /usr/share/mecab/dic/ipadic/Prefix.csv ... 221\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.adjv.csv ... 3328\n",
            "reading /usr/share/mecab/dic/ipadic/Interjection.csv ... 252\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.place.csv ... 72999\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.number.csv ... 42\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.org.csv ... 16668\n",
            "reading /usr/share/mecab/dic/ipadic/Filler.csv ... 19\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.csv ... 60477\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.name.csv ... 34202\n",
            "reading /usr/share/mecab/dic/ipadic/Others.csv ... 2\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.demonst.csv ... 120\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.proper.csv ... 27327\n",
            "reading /usr/share/mecab/dic/ipadic/Verb.csv ... 130750\n",
            "reading /usr/share/mecab/dic/ipadic/Auxil.csv ... 199\n",
            "reading /usr/share/mecab/dic/ipadic/Adj.csv ... 27210\n",
            "reading /usr/share/mecab/dic/ipadic/Postp.csv ... 146\n",
            "reading /usr/share/mecab/dic/ipadic/Conjunction.csv ... 171\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading /usr/share/mecab/dic/ipadic/matrix.def ... 1316x1316\n",
            "emitting matrix      : 100% |###########################################| \n",
            "\n",
            "done!\n",
            "update-alternatives: using /var/lib/mecab/dic/ipadic-utf8 to provide /var/lib/mecab/dic/debian (mecab-dictionary) in auto mode\n",
            "Setting up mecab (0.996-5) ...\n",
            "Compiling IPA dictionary for Mecab.  This takes long time...\n",
            "reading /usr/share/mecab/dic/ipadic/unk.def ... 40\n",
            "emitting double-array: 100% |###########################################| \n",
            "/usr/share/mecab/dic/ipadic/model.def is not found. skipped.\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.verbal.csv ... 12146\n",
            "reading /usr/share/mecab/dic/ipadic/Suffix.csv ... 1393\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.nai.csv ... 42\n",
            "reading /usr/share/mecab/dic/ipadic/Adverb.csv ... 3032\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.adverbal.csv ... 795\n",
            "reading /usr/share/mecab/dic/ipadic/Symbol.csv ... 208\n",
            "reading /usr/share/mecab/dic/ipadic/Adnominal.csv ... 135\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.others.csv ... 151\n",
            "reading /usr/share/mecab/dic/ipadic/Postp-col.csv ... 91\n",
            "reading /usr/share/mecab/dic/ipadic/Prefix.csv ... 221\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.adjv.csv ... 3328\n",
            "reading /usr/share/mecab/dic/ipadic/Interjection.csv ... 252\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.place.csv ... 72999\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.number.csv ... 42\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.org.csv ... 16668\n",
            "reading /usr/share/mecab/dic/ipadic/Filler.csv ... 19\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.csv ... 60477\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.name.csv ... 34202\n",
            "reading /usr/share/mecab/dic/ipadic/Others.csv ... 2\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.demonst.csv ... 120\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.proper.csv ... 27327\n",
            "reading /usr/share/mecab/dic/ipadic/Verb.csv ... 130750\n",
            "reading /usr/share/mecab/dic/ipadic/Auxil.csv ... 199\n",
            "reading /usr/share/mecab/dic/ipadic/Adj.csv ... 27210\n",
            "reading /usr/share/mecab/dic/ipadic/Postp.csv ... 146\n",
            "reading /usr/share/mecab/dic/ipadic/Conjunction.csv ... 171\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading /usr/share/mecab/dic/ipadic/matrix.def ... 1316x1316\n",
            "emitting matrix      : 100% |###########################################| \n",
            "\n",
            "done!\n",
            "Setting up mecab-jumandic (7.0-20130310-4) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1.3) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/python3.6/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
            "\n",
            "                            \n",
            "Collecting mecab-python3==0.6\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e3/7f/f98371035a0171abf95f9893eabf915f8a3199d005fed3cd69cc122fed40/mecab-python3-0.6.tar.gz (41kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 7.7MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: mecab-python3\n",
            "  Building wheel for mecab-python3 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for mecab-python3: filename=mecab_python3-0.6-cp36-cp36m-linux_x86_64.whl size=155469 sha256=3dfaddc28a65c0ab5e3cbfeb9464d6ef85583d97292750b23bed03b6e73dd863\n",
            "  Stored in directory: /root/.cache/pip/wheels/4d/51/5b/987888cacaf8bb25982ef4569261f68debe85b7587c5563c79\n",
            "Successfully built mecab-python3\n",
            "Installing collected packages: mecab-python3\n",
            "Successfully installed mecab-python3-0.6\n",
            "Collecting japanize_matplotlib\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/aa/85/08a4b7fe8987582d99d9bb7ad0ff1ec75439359a7f9690a0dbf2dbf98b15/japanize-matplotlib-1.1.3.tar.gz (4.1MB)\n",
            "\u001b[K     |████████████████████████████████| 4.1MB 16.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from japanize_matplotlib) (3.2.2)\n",
            "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.6/dist-packages (from matplotlib->japanize_matplotlib) (1.19.5)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->japanize_matplotlib) (1.3.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->japanize_matplotlib) (0.10.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->japanize_matplotlib) (2.4.7)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->japanize_matplotlib) (2.8.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from cycler>=0.10->matplotlib->japanize_matplotlib) (1.15.0)\n",
            "Building wheels for collected packages: japanize-matplotlib\n",
            "  Building wheel for japanize-matplotlib (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for japanize-matplotlib: filename=japanize_matplotlib-1.1.3-cp36-none-any.whl size=4120276 sha256=c6bd948c7cd4187a1a73466dc3f5c2283e886ce21b185484abf05e5bf2fde85a\n",
            "  Stored in directory: /root/.cache/pip/wheels/b7/d9/a2/f907d50b32a2d2008ce5d691d30fb6569c2c93eefcfde55202\n",
            "Successfully built japanize-matplotlib\n",
            "Installing collected packages: japanize-matplotlib\n",
            "Successfully installed japanize-matplotlib-1.1.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NWWVj3Bg9XzT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3f56e994-23c2-4c5a-d524-2fe3a77f0aac"
      },
      "source": [
        "import numpy as np\n",
        "import os\n",
        "import time\n",
        "import MeCab\n",
        "\n",
        "import preprocess_utils\n",
        "import model\n",
        "import weight_utils\n",
        "\n",
        "import tensorflow.keras as keras\n",
        "import tensorflow as tf\n",
        "print(tf.__version__)\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import japanize_matplotlib\n",
        "%matplotlib inline"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.4.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uDKA-HHP9h23"
      },
      "source": [
        "# 日英翻訳データ ダウンロード"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J4T1VwIz9X2G"
      },
      "source": [
        "!wget http://www.manythings.org/anki/jpn-eng.zip\n",
        "!unzip ./jpn-eng.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bT9Sr5Le9krj"
      },
      "source": [
        "# データ読み込み"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nYD-xKxPG-1D"
      },
      "source": [
        " `corpus_path = './jpn.txt'` -> `corpus_path = './DATA/kesen3_ex.tsv'`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R7tKLO4r9X4m",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ed0f47dc-2fa4-4578-eb39-fcab85469763"
      },
      "source": [
        "dataset = preprocess_utils.CreateData(\n",
        "    corpus_path = './DATA/kesenngo.tsv',\n",
        "    do_shuffle=True,\n",
        "    seed_value=123,\n",
        "    split_percent=0.95 # 学習データの割合\n",
        ")\n",
        "\n",
        "\n",
        "train_source, train_target, test_source, test_target, train_licence, test_licence = dataset.split_data()\n",
        "test_source[-1]= \"佐藤ではない\"\n",
        "test_target[-1]= \"\"\n",
        "test_licence[-1]= \"大船渡市22\"\n",
        "print('**** Amount of data ****')\n",
        "print('train_source： ', len(train_source))\n",
        "print('train_target： ', len(train_target))\n",
        "print('test_source： ', len(test_source))\n",
        "print('test_target： ', len(test_target))\n",
        "print('\\n')\n",
        "print('**** Train data example ****')\n",
        "print('Source Example： ', train_source[0])\n",
        "print('Target Example： ', train_target[0])\n",
        "print('Licence： ', train_licence[0])\n",
        "print('\\n')\n",
        "print('**** Test data example ****')\n",
        "print('Source Example： ', test_source[-1])\n",
        "print('Target Example： ', test_target[-1])\n",
        "print('Licence： ', test_licence[-1])"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "**** Amount of data ****\n",
            "train_source：  8654\n",
            "train_target：  8654\n",
            "test_source：  456\n",
            "test_target：  456\n",
            "\n",
            "\n",
            "**** Train data example ****\n",
            "Source Example：  本家\n",
            "Target Example：  ほんけ\n",
            "Licence：  気仙沼市\n",
            "\n",
            "\n",
            "**** Test data example ****\n",
            "Source Example：  佐藤ではない\n",
            "Target Example：  \n",
            "Licence：  大船渡市22\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ASlKXytP9ndi"
      },
      "source": [
        "# 前処理"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cnl0wqW59X64"
      },
      "source": [
        "BATCH_SIZE = 64 # バッチサイズ\n",
        "MAX_LENGTH = 60 # シーケンスの長さ\n",
        "USE_TPU = False # TPUを使うか\n",
        "BUFFER_SIZE = 50000"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B_dbI1Vs9X9i"
      },
      "source": [
        "train_dataset = preprocess_utils.PreprocessData(\n",
        "    mecab = MeCab.Tagger(\"-Ochasen\"),\n",
        "    source_data = train_source,\n",
        "    target_data = train_target,\n",
        "    max_length = MAX_LENGTH,\n",
        "    batch_size = BATCH_SIZE,\n",
        "    test_flag = False,\n",
        "    train_dataset = None,\n",
        ")\n",
        "\n",
        "train_dataset.preprocess_data()"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-YskuOvz9YAC"
      },
      "source": [
        "test_dataset = preprocess_utils.PreprocessData(\n",
        "    mecab = MeCab.Tagger(\"-Ochasen\"),\n",
        "    source_data = test_source,\n",
        "    target_data = test_target,\n",
        "    max_length = MAX_LENGTH,\n",
        "    batch_size = BATCH_SIZE,\n",
        "    test_flag = True,\n",
        "    train_dataset = train_dataset\n",
        ")\n",
        "\n",
        "test_dataset.preprocess_data()"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UNd20r7NtSRM"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UhJ1_jlY9rQA"
      },
      "source": [
        "# バッチ作成"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aMKDhiii9YCZ"
      },
      "source": [
        "if USE_TPU:\n",
        "  tpu_grpc_url = \"grpc://\" + os.environ[\"COLAB_TPU_ADDR\"]\n",
        "  tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu_grpc_url)\n",
        "  tf.config.experimental_connect_to_cluster(tpu_cluster_resolver)\n",
        "  tf.tpu.experimental.initialize_tpu_system(tpu_cluster_resolver)    \n",
        "  strategy = tf.distribute.experimental.TPUStrategy(tpu_cluster_resolver)\n",
        "\n",
        "trainset = tf.data.Dataset.from_tensor_slices((train_dataset.source_vector, train_dataset.target_vector))\n",
        "trainset = trainset.map(lambda source, target: (tf.cast(source, tf.int64), tf.cast(target, tf.int64))).shuffle(buffer_size=BUFFER_SIZE).batch(BATCH_SIZE).prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n",
        "\n",
        "if USE_TPU:\n",
        "  trainset = strategy.experimental_distribute_dataset(trainset)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jgUNxjQK9YEu"
      },
      "source": [
        "if USE_TPU:\n",
        "  PREDICT_BATCH_SIZE = 8\n",
        "  testset = tf.data.Dataset.from_tensor_slices((test_dataset.source_vector, test_dataset.target_vector))\n",
        "  testset = testset.map(lambda source, target: (tf.cast(source, tf.int64), tf.cast(target, tf.int64))).shuffle(buffer_size=50000).batch(PREDICT_BATCH_SIZE).prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n",
        "  testset = testset.take(1)\n",
        "  testset = strategy.experimental_distribute_dataset(testset)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nwt7j74P9uue"
      },
      "source": [
        "# モデル定義"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PanDUL9e9wm7"
      },
      "source": [
        "num_layers=4 # レイヤー数\n",
        "d_model=64 # 中間層の次元数\n",
        "num_heads=4 # Multi Head Attentionのヘッド数\n",
        "dff=2048 # Feed Forward Networkの次元数\n",
        "dropout_rate = 0.1 # ドロップアウト率\n",
        "\n",
        "source_vocab_size = max(train_dataset.source_token.values()) + 1 # source文の語彙数\n",
        "target_vocab_size = max(train_dataset.target_token.values()) + 1 # target文の語彙数"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Lgupz9a9wpt"
      },
      "source": [
        "# 重み初期化\n",
        "def initialize_weight(checkpoint_path, optimizer, transformer, max_length, batch_size, use_tpu=False):\n",
        "\n",
        "  if os.path.exists(checkpoint_path+'.pkl'):\n",
        "    if use_tpu:\n",
        "      number_of_tpu_cores = tpu_cluster_resolver.num_accelerators()['TPU']\n",
        "      initialize_source, initialize_target = [[1]*max_length]*number_of_tpu_cores, [[1]*max_length]*number_of_tpu_cores\n",
        "      initialize_set = tf.data.Dataset.from_tensor_slices((initialize_source, initialize_target))\n",
        "      initialize_set = initialize_set.map(lambda source, target: (tf.cast(source, tf.int64), tf.cast(target, tf.int64))\n",
        "          ).shuffle(buffer_size=BUFFER_SIZE).batch(batch_size).prefetch(\n",
        "              buffer_size=tf.data.experimental.AUTOTUNE\n",
        "          )\n",
        "      initialize_set = strategy.experimental_distribute_dataset(initialize_set)\n",
        "\n",
        "      for inp, tar in initialize_set:\n",
        "        distributed_train_step(inp, tar)\n",
        "\n",
        "    else:\n",
        "      initialize_set = tf.ones([batch_size, max_length], tf.int64)\n",
        "      train_step(initialize_set, initialize_set)\n",
        "    \n",
        "    try:\n",
        "      weight_utils.load_weights_from_pickle(checkpoint_path, optimizer, transformer)\n",
        "    except:\n",
        "      print('Failed to load checkpoints.')\n",
        "\n",
        "  else:\n",
        "    print('No available checkpoints.')"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MRZ7rE6G9zkK"
      },
      "source": [
        "# 予測"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ENG4IErgG6aA"
      },
      "source": [
        "checkpoints/gpu/model -> /checkpoints_EX/gpu/model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TE5AjytD9wsz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5facad07-3c68-4648-fe03-fd35e272437a"
      },
      "source": [
        "# Transformer\n",
        "transformer = model.Transformer(num_layers, d_model, num_heads, dff,\n",
        "                          source_vocab_size, target_vocab_size, \n",
        "                          pe_input=source_vocab_size, \n",
        "                          pe_target=target_vocab_size,\n",
        "                          rate=dropout_rate)\n",
        "\n",
        "# Learning Rate\n",
        "learning_rate = model.CustomSchedule(d_model)\n",
        "\n",
        "# Optimizer\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, \n",
        "                                     epsilon=1e-9)\n",
        "\n",
        "# Loss\n",
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')\n",
        "\n",
        "# Loss Function\n",
        "def loss_function(real, pred):\n",
        "  mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
        "  loss_ = loss_object(real, pred)\n",
        "\n",
        "  mask = tf.cast(mask, dtype=loss_.dtype)\n",
        "  loss_ *= mask\n",
        "  return tf.reduce_mean(loss_)\n",
        "\n",
        "# Metrics\n",
        "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
        "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n",
        "\n",
        "# Checkpoint\n",
        "checkpoint_path = \"/content/drive/My Drive/Colab Notebooks/Transformer/checkpoints_EX/gpu/model\" #/checkpoints/gpu/model\n",
        "\n",
        "train_step_signature = [\n",
        "    tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n",
        "    tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n",
        "]\n",
        "@tf.function(input_signature=train_step_signature)\n",
        "def train_step(inp, tar):\n",
        "  tar_inp = tar[:, :-1]\n",
        "  tar_real = tar[:, 1:]\n",
        "  \n",
        "  enc_padding_mask, combined_mask, dec_padding_mask = model.create_masks(inp, tar_inp)\n",
        "  \n",
        "  with tf.GradientTape() as tape:\n",
        "    predictions, _ = transformer(inp, tar_inp, \n",
        "                                 True, \n",
        "                                 enc_padding_mask, \n",
        "                                 combined_mask, \n",
        "                                 dec_padding_mask)\n",
        "    loss = loss_function(tar_real, predictions)\n",
        "\n",
        "  gradients = tape.gradient(loss, transformer.trainable_variables)    \n",
        "  optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n",
        "  \n",
        "  train_loss(loss)\n",
        "  train_accuracy(tar_real, predictions)\n",
        "\n",
        "# Initialize Weight\n",
        "initialize_weight(checkpoint_path, optimizer, transformer, MAX_LENGTH, BATCH_SIZE, use_tpu=USE_TPU)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Load checkpoints successfully.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m9xw1D1D9YHw"
      },
      "source": [
        "def predict(input_vec):\n",
        "\n",
        "  encoder_input = np.array([input_vec])\n",
        "\n",
        "  decoder_input = [max(train_dataset.target_token.values())-2]\n",
        "  output = tf.expand_dims(decoder_input, 0)\n",
        "\n",
        "  for i in range(train_dataset.max_length):\n",
        "    enc_padding_mask, combined_mask, dec_padding_mask = model.create_masks(\n",
        "        encoder_input, output)\n",
        "\n",
        "\n",
        "    predictions, attention_weights = transformer(encoder_input, \n",
        "                                                  output,\n",
        "                                                  False,\n",
        "                                                  enc_padding_mask,\n",
        "                                                  combined_mask,\n",
        "                                                  dec_padding_mask)\n",
        "    \n",
        "    predictions = predictions[: ,-1:, :]\n",
        "\n",
        "    predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n",
        "\n",
        "    if predicted_id == max(train_dataset.target_token.values())-1:\n",
        "      return tf.squeeze(output, axis=0), attention_weights\n",
        "    \n",
        "    output = tf.concat([output, predicted_id], axis=-1)\n",
        "\n",
        "  return tf.squeeze(output, axis=0), attention_weights"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hUaGS8LD9YKo"
      },
      "source": [
        "def plot_attention_weights(attention, input_source, result, layer):\n",
        "  fig = plt.figure(figsize=(16, 8))\n",
        "  \n",
        "  sentence = tf.cast(input_source, tf.int64)\n",
        "  attention = attention[layer][0,:, :, :len(input_source)]\n",
        "  \n",
        "  for head in range(attention.shape[0]):\n",
        "    ax = fig.add_subplot(2, 4, head+1)\n",
        "    \n",
        "    ax.matshow(attention[head][:-1, :], cmap='viridis')\n",
        "\n",
        "    fontdict = {'fontsize': 10}\n",
        "    \n",
        "    ax.set_xticks(range(len(sentence)))\n",
        "    ax.set_yticks(range(len(result)))\n",
        "    \n",
        "    ax.set_ylim(len(result)-1.5, -0.5)\n",
        "        \n",
        "    tmp_list = []\n",
        "    for i in sentence:\n",
        "      try:\n",
        "        tmp_list.append(train_dataset.source_index[i.numpy()])\n",
        "      except:\n",
        "        pass\n",
        "\n",
        "    ax.set_xticklabels(tmp_list, fontdict=fontdict, rotation=90)\n",
        "    \n",
        "    ax.set_yticklabels([train_dataset.target_index[i.numpy()] for i in result \n",
        "                        if i < max(train_dataset.target_token.values()) - 1], \n",
        "                       fontdict=fontdict)\n",
        "    \n",
        "    ax.set_xlabel('Head {}'.format(head+1))\n",
        "  \n",
        "  plt.tight_layout()\n",
        "  plt.show()"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "voh_q3jH-Jo9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 268
        },
        "outputId": "7961d529-430f-480e-b026-d49ec7cced4a"
      },
      "source": [
        "# 翻訳\n",
        "n = 0# 任意のテストデータを指定\n",
        "print(\"Input:\", ' '.join([train_dataset.source_index[i] for i in test_dataset.source_vector[n][:np.argmax(test_dataset.source_vector[n], 0)+1]]))\n",
        "\n",
        "result, attention = predict(test_dataset.source_vector[n][:np.argmax(test_dataset.source_vector[n], 0)+1])\n",
        "txt = \"\"\n",
        "\n",
        "for i in result[1:]:\n",
        "  txt+=train_dataset.target_index[i.numpy()]\n",
        "print(\"Output:\", txt)\n",
        "\n",
        "# Attention Weightをプロット\n",
        "plot_attention_weights(attention, test_dataset.source_vector[n][:np.argmax(test_dataset.source_vector[n], 0)+1], result[1:], \"decoder_layer4_block2\")"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: <start> 行っ た なあ <end>\n",
            "Output: 行ったよねー\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABHgAAADZCAYAAABb/YcbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAbfElEQVR4nO3deZBtB10n8O8vb8u+YZA1gIIRQcQCCyMuYZVQQHDcGHUGRUCnZnBDhBmBcrRmmBkj6CjgBFcUBxhqVIxIwbC4QSgjJlFRUDQJKmtAQtb38vKbP7oftEnnpfv1Pffcc/rzqerKPbe7z/n9zu37fff8cu651d0BAAAAYLqOG7sAAAAAAHbGgAcAAABg4gx4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4gx4AAAAACZuVwx4quopVfWYsesAdgeZAyyb3AGWSebAato7dgFL8qgkleRtYxcC7AoyB1g2uQMsk8yBFVTdPXYNg6qqxyU5kOTUJB/s7veMXBIwYzIHWDa5AyyTzIHVtRveovXk7r44yeuTfPvYxcCqqKrnVtX3jF3HDMkc2ITMGZTcgU3IncHIHNjEKmTOrAc8VfXlSf4iSbr7liRXV9UDxq0KxldV+5N8fpJzqqrGrmcuZA5sTuYMR+7A5uTOMGQObG5VMmfWA54kX5TkVzcsvyrJg0eqBVbJdyR5TZLfSvLNI9cyJzIHNidzhiN3YHNyZxgyBza3Epkz6wFPd78uyddtWP50kk+MVxGsjId19+Xd/QdJHj12MXMhc+AOyZyByB24Q3JnADIH7tBKZM6sBzzrXnDkRlXtS/KTI9YCo6uqpyT53Q13/X5VeeGzODIHNpA5SyF3YAO5MziZAxusUubM9mPSq+qrk/xakrtV1d9l7WP8bk3y9lELg/F9rLsv2bD8+iQPG6uYuZA5cIdkzkDkDtwhuTMAmQN3aGUyZzd8TPr3dvfPj10HsDvIHGDZ5A6wTDIHVtduGPD8dndfMHYdsAqq6sNJbvuk7yTXdPdDRihpdmQOfI7MWQ65A58jd4Ync+BzVi1zZvsWrQ0urqrnJfm9JAeTpLs/MG5JMI7uvnuSVNWLkrylu99TVU9M8ohxK5sVmQPrZM7SyB1YJ3eWQubAulXLnN1wBs87bnNXd7eLrLGrVdWbuvuJG5Z/v7u/7mi/w9bIHLg9mTMsuQO3J3eGI3Pg9lYlc2Z/Bk93P2rsGmAFnV5VD+3uy6rqEdkFWbAsMgc2JXMGJHdgU3JnIDIHNrUSmbMbzuD5/CTfluSUrF3p/d7d/cxxq4JxVdVDkrwqyb2T/H2SZ3X3+8atah5kDtyezBmW3IHbkzvDkTlwe6uSOccte4MjeG2STyR5apJ/SHL5uOUwB1V196p6XVW9u6p+oapOHbum7ejuK7r7Ed19j+5+pBc8CyVzWDiZw52QOyzU1DMnkTsDkzkslMxZnN0w4DnQ3b+W5Nru/sUkjx27oGNRVY+oqndV1Xur6qVj17MbVdWJG8LmFUle0t3nJnljkv89XmXbV1VfWlUvq6pfOvI1dk0zInNYCJnDNkw+d2TO+OaUOYncGZjMYcdkzjB2w3tRr66q85N8uKq+P8l9R67nWP14kid39zVV9ZNV9W3d/RtjF7XLPDDJeVX18SQ3dPdlSdLdb6yqp1TVOd39/nFL3LJfS3Jhkg+PXcgMyRwWReawVXPIHZkzvjllTiJ3hiRzWASZM4DdcAbPdyV5b5LnZy18nj1qNcduf3dfs377JUm+Zcxidqm/SvLQJH+e5Eur6jer6pHr3/ulJDeOVtn2fby7f72733bka+yCZkTmsCgyh62aQ+7InPHNKXMSuTMkmcMiyJwB7IYBz3O6+6PdfXV3/2CSrxi7oGN0a1VVknT3J5OcPnI9u9E3JblXkkNJ/jrJjyR5flX9aNZOVb16zOK26Q+r6vur6ouOfI1d0IzIHBZF5rBVc8gdmTO+OWVOIneGJHNYBJkzgNm+Rauq7pbknCRPq6pL1u/en+TfJfm50Qo7dh9I8mVJLltffsmItexWtyS5ez73qQG/kOTSJF+z/v13jFTXsTjy8ZZPXf9vJ3n0SLXMgsxhADKHo5pZ7sic8c0pcxK5s3AyhwWTOQOY7cekV9X9krw4yROS/F7WPsKvk/xBd//KiKUdk6r69iT37O7/MXYtu1lVvaO7H1VVxyd5Q5Ify9o/EK/u7qce9ZeZNZnDEGQORzOn3JE5q0HmcDQyh0WTOYs32wHPEVX1ld19yZ3/5GqrqpOTHNfd145dy25WVd/e3a9Zv316kp9Mcr8kf9fdk3n/cVXdN8nLkpyVtfdPn93dk7ta/SqSOSySzGEr5pA7Mmc1zCVzErkzJJnDosicAerYBQOef5vk/2VtR/9skt/s7peNW9X2VdWTuvvisetgTVUd6O6bx67jWFXVW5I8L2sh9Jis/Z+Xrzn6b7EVMochyByOZg65I3NWy9QzJ5E7Q5I5LJrMWZzdcJHlZ3T3PyV5ZpLHJ/nWkevZsqr65Q2LPzRaIfwLVXVBkt/YsPy9t3mspmB/d1+eJL025b115HrmROawUDKHLZhk7sic1TSTzEnkzpBkDgsjcxZrthdZ3uCEqjo/yUeSHEwypcngvTbc/rKqetP67Vr/urS7X7j8so5dVX1Ld79+w/K7u/vcMWvajqp6UNb+QXji+vLnJfnBJI882u+toI9U1QuTnFRV353kQ2MXNCMyZ4XInJUhc4Y11dyROStmRpmTyJ0hyZwVIXNWykpkzqzP4Kmq47L2/rdHZe1UqbOTvHzUorZn4/vnrujuJ65/nd/dT0jyVWMVtgM/cJvlqQ0ZP5jkG5M8u6remuQ9Wftov9+pqseNWtkWrT8vnpHkQJKPZe19rt8zalEzIXNWkswZmcwZ1sRzR+asnslnTiJ3hiRzVo7MWQGrlDmzHvBkbfp3ZXf/SHffkOTUJGeOXNN21Ibbm10s6W+XVcgCHayqPRuWD41WybE5J2sXzHpZdz8uyeVJvi7JBUmeM2plW/eDSe7a3S/q7icneX2SfzNyTXMhc1aPzBmfzBnWlHNH5qyeOWROIneGJHNWi8xZDSuTOXMf8PxSkmdtWP7OJL86TinH5PINt+u235zSlcWr6ulV9cVJ3pfkC2vNU7J2auckVNWXJDkt66cQrrs6yaOzdhrh34xR1zGY+vNilU1938qcFSJz2KIp71+Zs0JmlDnJtJ8Xq27K+1bmrBCZM4ypncK1Ld39qarqqjotyYlJPtXdN45d1zb8zySpqh/P2kfGTdL6lfbvm+TTSb6vu2+pqhOTfGXWLs42Fd+U5AlJTq6qd3X325O8IMn3JTmc5EfHLG6rZvC8WFkz2LcyZ7XIHO7UxPevzFkts8icZPLPi5U28X0rc1aLzBnAbviY9Ptk7X19Jyb5+e7+xMglbUlVnZDktUn+T5Irk1yY5JQk/3jkR7J2ge7Hj1LgNlTVM5M8OcmDklyc5A3d/UfjVnVs1k+BvCTJ27L2Pt3nd/e7x61q+6b6vJiCqe5bmbOaZA5bMcX9K3NW01wyJ5nm82IqprhvZc5qkjmLN+szeJKku6+qqrsnuXkK4XNEd99YVU9N8sqsvUf0q5O8Osm7u/tnRy1um7r7F6rqiqydbvfWJE+rqguTvCnJK7v746MWuD2d5IPd/YKqOjvJa6rqx7r7bWMXth1TfV5MwVT3rcxZWTKHOzXF/StzVtYsMieZ5vNiKqa4b2XOypI5Czb7M3iSz07TDnb3h8euZbvWr8j9gO5+f1VVkvO6+x1j17VdVfWVSR7W3S9fX96b5GlJntbdTxq1uB2oquO6+9ax6zgWU35erLop71uZs9pkDndkqvtX5qy2KWdOMt3nxRRMdd/KnNUmcxZQw24Y8AAAAADM2dw/RQsAAABg9nbVgKeqJvPRd3dkDj0k8+hjDj0k8+ljFc1l386hjzn0kMyjjzn0sMrmsH/n0EMyjz7m0EMynz5W0Rz27Rx6SObRxxx6SMbtY1cNeJLM4Q9mDj0k8+hjDj0k8+ljFc1l386hjzn0kMyjjzn0sMrmsH/n0EMyjz7m0EMynz5W0Rz27Rx6SObRxxx6SEbsY7cNeAAAAABmZ2Uusrz31BN7311PH3Qbh6+9IXtOPXGw9d/thGsHW/cRn/nkoZxy5r5Bt3HNX+wfdP1Jcig3Z18ODLqNm88+adD1H77uuuw5+eRBt5Ea/vl5+Lrrs+fkYffVwav+8RPdfdagGzkG+/af1MefcMZg6z908Prs2z/svj10Ug26/iQ5fMP12XPisH0c+MgNg67/YG7O/oEzJ0n6xOMHXf+hW67Pvr3DPhYZ+E/q4KHrs3/fsD3cdNM/5+Ch64d/cmzTvuNP6v0nnTnoNm656frsPX64/bvnpsODrfuIQ7fckH17h3u9liSHTt0z6PqT5JYbrs/egbPzuIODrn7wv6ckg2dOMnwfN1/3ydxy0+plzp5TTuq9Zw19fHV99pw63L59wMnDf+L3pz55a844c9jzHq7+84GPGbKk46t7DpsHt15/fY47aeDMWcIpLss4Tjz4oX/Y9Phq76Bb3YZ9dz09X/jSZ45dxo48/4FvGbuEhXj1Ofceu4SF+JvnP2LsEnasTxj+hfQyXP2MF1w1dg2bOf6EM/Kwc58zdhk78uFzhx/ILsN9//t7xy5hIQ4/9JyxS9ixPm7ljlG27U/+7BVjl7Cp/SedmQef/wNjl7Ejp33gurFLWIgPPe7UsUtYiFOvnOynCX/Wrfumnznvu/hlY5ewqb1nnZ57/MS/H7uMHXnD1/7c2CUsxHPu88ixS1iIv/v+c8cuYccOnzj93EySq/7D8zY9vvIWLQAAAICJM+ABAAAAmDgDHgAAAICJM+ABAAAAmDgDHgAAAICJM+ABAAAAmDgDHgAAAICJM+ABAAAAmDgDHgAAAICJM+ABAAAAmDgDHgAAAICJM+ABAAAAmDgDHgAAAICJM+ABAAAAmDgDHgAAAICJM+ABAAAAmDgDHgAAAICJM+ABAAAAmLgtD3iq6sQhCwHYSOYAyyRzgGWTO8Ci7d3KD1XVcUleW1Xfl+Q5Sb5sw7f/sbufPkRxwO4kc4BlkjnAsskdYAh3OuCpqj1JXpHkDd19ZZLnDl0UsHvJHGCZZA6wbHIHGMpW3qL1gCR/lOTvq+onBq4HQOYAyyRzgGWTO8Ag7vQMnu7+66r6YNZC6Ceq6p0bvv2QJFckeWmSdyS5IMlXJ/mirA2PDiV5UXdfsuC6gZmSOcAyyRxg2eQOMJStvEXruCS/muTU7r44ycUbvvfm7n7C+u3TktwryUVJLu/uw1V1VpI3Jjn3Dtb97CTPTpJ9Z522w1aAORgyc9Z/77O5c+D40wfrA5iGZWbO/hPPGKwPYDqWdXy15y6Or2C32cpFls9L8p4kdz3aD3X3p5P8t9vc9/GqOuEov3NR1gIrJ9z/Hr2FWoD5Oy8DZc76z3w2d0457V5yBzgvS8qck+5yb5kDJEs6vjrwBfeUObDL3Ok1eLr77d39M9tdcVXdpaouTPLOYykM2J1kDrBMMgdYNrkDDGVLH5N+FLcbEFXVE5L8xySfTvL6JK/Z4TYAjpA5wDLJHGDZ5A5wzI5pwFNVr8ra1d/fddvvdfebk7x5h3UBfJbMAZZJ5gDLJneARdjygKe7H7vh9rOGKQdgjcwBlknmAMsmd4BFu9Nr8AAAAACw2gx4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4gx4AAAAACbOgAcAAABg4vaOXcARtx7ck+s+dOrYZezIi6/85rFLWIh+eY9dwmIcHruAnasb9oxdwqzVtTdk31v/dOwyduTst8zj+fp7/3TZ2CUsxBPOvmXsEnasb53B39ThG8euYFN7b7glZ1z2ybHL2JHD7/vA2CUsxF++cR6Z8/hvfPrYJezYcYem/4Jtz40r2sOtlVtvXJnDvWNy/v997tglLMRJz53HeRW3nDb91zlzN4+/NAAAAIBdzIAHAAAAYOIMeAAAAAAmzoAHAAAAYOIMeAAAAAAmzoAHAAAAYOIMeAAAAAAmzoAHAAAAYOIMeAAAAAAmzoAHAAAAYOIMeAAAAAAmzoAHAAAAYOIMeAAAAAAmzoAHAAAAYOIMeAAAAAAmzoAHAAAAYOIMeAAAAAAmzoAHAAAAYOIMeAAAAAAmbuEDnqp6VlU9fdHrBbgjcgdYJpkDLJPMAbZqYQOeqjq3qh6R5MlJvqCqvmtR6wbYjNwBlknmAMskc4Dt2rvAdT0sySlJHpzkV5L81QLXDbAZuQMsk8wBlknmANuysAFPd/9cVT08yd2TXJzkZ6rqS5IcSPLT3f3aRW0LIJE7wHLJHGCZZA6wXYs8gydJnpbkDd19sKp+JskHu/vQgrcBsJHcAZZJ5gDLJHOALVvkNXj2JfnaJH9YVScl+cCdhU9VPbuqLq2qSw9fd92iSgF2iZ3mzqHcvJQ6gXnYaeYcPHzDUuoE5mHHx1efuX4pdQKrY5GfovW0JG/q7k7ywqxdDOyouvui7n54dz98z8knL7AUYJfYUe7sy4HBCwRmZUeZs3/PiYMXCMzKzo6vTjlp8AKB1bLIAc+/TvLK9dsXJvnOqrpkfYL8ngVuB+AIuQMsk8wBlknmANuyyIssP3HD7WuSfMOi1g2wGbkDLJPMAZZJ5gDbtcgzeAAAAAAYgQEPAAAAwMQZ8AAAAABMnAEPAAAAwMQZ8AAAAABMnAEPAAAAwMQZ8AAAAABMnAEPAAAAwMQZ8AAAAABMnAEPAAAAwMQZ8AAAAABMnAEPAAAAwMQZ8AAAAABMnAEPAAAAwMQZ8AAAAABMnAEPAAAAwMQZ8AAAAABMnAEPAAAAwMQZ8AAAAABMnAEPAAAAwMTtHbuAz9rTySm3jF3Fjuy/8sDYJSzE4f09dgkLceiMw2OXsGN1wvR7WGV96ok5+FUPG7uMHTl8/Dzm9Oef/8CxS1iImx9z0tglkKTf9Udjl7Cpg6ftzYee+Hljl7EzU69/3dffY+wKFuPa7zhh7BJ27Ma7TP/fsUNX7Rm7hE3Vocr+j67O4d6xOPDPNXYJC3H/C/5m7BIW4s8vuf/YJezcrWMXMKzpJyoAAADALmfAAwAAADBxBjwAAAAAE2fAAwAAADBxBjwAAAAAE2fAAwAAADBxBjwAAAAAE2fAAwAAADBxBjwAAAAAE2fAAwAAADBxBjwAAAAAE2fAAwAAADBxBjwAAAAAE2fAAwAAADBxBjwAAAAAE2fAAwAAADBxBjwAAAAAE2fAAwAAADBxBjwAAAAAE2fAAwAAADBxgw14quqiqrr/UOsH2EjmAMskc4BlkzvAnRnyDJ7DSVJVB6rq7AG3A5DIHGC5ZA6wbHIHOKohBzwHk/x8ktcmeeqA2wFIZA6wXDIHWDa5AxzV3gHXfVOSH+ruKwbcBsARMgdYJpkDLJvcAY5qyDN4PpPkzKP9QFU9u6ourapLD3/m+gFLAXaBO82c5F/mzqGDcgc4ZtvOnMM3yhxgR7Z3fHW9zIHdZqFn8FTVyUlekOQrktw7yTlV9dHu/qvNfr67L0pyUZIcuN+9epG1APO33cxJ/mXunHKa3AG2bqeZc8Ld7i1zgG3ZyfHV8feUObDbLPoMnu9O8rEkT0ryi1mbMr+kql5fVXdf8LYAZA6wTDIHWDa5A2zZoq/B8+Ykr0zyTUn2J/mG7v5wVT0kyTUL3haAzAGWSeYAyyZ3gC1b6ICnu9+f5NGb3O9CYMDCyRxgmWQOsGxyB9iOIS+yDAAAAMASGPAAAAAATJwBDwAAAMDEGfAAAAAATJwBDwAAAMDEGfAAAAAATJwBDwAAAMDEGfAAAAAATJwBDwAAAMDEGfAAAAAATJwBDwAAAMDEGfAAAAAATJwBDwAAAMDEGfAAAAAATJwBDwAAAMDEGfAAAAAATJwBDwAAAMDEGfAAAAAATJwBDwAAAMDEGfAAAAAATFx199g1JEmq6uNJrhp4M5+X5BMDb2Noc+ghmUcfc+ghWU4f9+nuswbexrYtIXf8jayOOfSQzKMPmTMsfyOrYw59zKGHZPg+ZM60zaGHZB59zKGHZMTXOisz4FmGqrq0ux8+dh07MYceknn0MYcekvn0sYrmsm/n0Mccekjm0cccelhlc9i/c+ghmUcfc+ghmU8fq2gO+3YOPSTz6GMOPSTj9uEtWgAAAAATZ8ADAAAAMHG7bcBz0dgFLMBK9FBVH7nN8mOr6le2sYpN+6iqd1bVF29y/zOr6k1V9cfbq3RQK/FYLMBc+lhFc9m3o/cxQuY8p6ouqap3V9UrqmpV/r0c/bFYgDn0sMrmsH9H72EBmZNs0sdmmVNVx1XVT1XVH1fVFVX1ku1XPJjRH4sFmUsfq2gO+3Ylelj2a50N3//FY8i3oazEY7EAo/WxKi9Yl6K7J/8HM4cekmPq46okL0iyZ4ByjskufizYorns2zn0sZ0equpBSZ6c5JHdfW6Ss5I8aajatmO3PRZs3xz27xx6SLbVxwOS/FN3PzLJlyf52qr6iuEq27pd+FiwTXPYt3PoITm2PqrqqUn2D1DOMdnNj8Wi7KoBD8tRVXerqt+tqt+vqt+pqrus3//MqnpvVf1JVX3r+n2nV9XFVfWOqnpVklM2W2d3vzXJtcvrApiKRWdOd/9lkqd09+H1u/YmuXFZ/QCrbYDMeX93/9T64plJDie5ckntABMwxPFVVX1+kh9O8l+W1giD2zt2AUzWmVX1zg3LZyT5s/XbFyZ5XXe/uqouSPKiJD+Q5OYk52btLJy3JXldkv+U5O3d/dL1oLp8SfUD07LUzOnum6rq9CSvSHLZ+pAZ2D2W/jpnfXsPTPK87v74YtsBJmDZufO/sjbguWnRjTAeAx6O1Se7+7wjC1X12CTfsb740CT3qapnZO0ssY+tX7/ivknemuTWrAVWkjwoyWuTpLuvqaq/XUr1wNQsNXOq6sFJfirJi7v7PQvvBlh1S3+d093nVdUZSX63qq7u7ncutCNg1S0td6rqe5K8r7svqar7DtEM4zDgYQhXJPnl7n5rVR3I2vvJH5LkgqxNmE9J8gcbfvbxSd5bVWdnLZAAtmOhmVNVZyX56STf2N2fXkL9wLQsOnMem+T47r64uz9VVVclOX0JfQDTsejjq69PcqCqfivJiUm+uKou7O4fHrgPBmbAwxB+KMlFVfXCrE2Y/3PWTi/8aJK3r9++cj2c/muSX6+1T8e6Osl7xykZmLBFZ863Jrlfkt+uqiP3/cZcLvwH7NiiM+eyJC+vqhdn7fo7f5rkjYN3AUzJQnOnu//VkdvrZ/D8mOHOPFR3j10DAAAAADvgU7QAAAAAJs6ABwAAAGDiDHgAAAAAJs6ABwAAAGDiDHgAAAAAJs6ABwAAAGDiDHgAAAAAJs6ABwAAAGDi/j8HzBA3ZesQywAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 1152x576 with 4 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 202
        },
        "id": "JaqqbogCAaKy",
        "outputId": "7972f68d-7566-4822-e8b2-c693abc700a6"
      },
      "source": [
        "# 翻訳\r\n",
        "n = 455# 任意のテストデータを指定\r\n",
        "print(\"Input:\", ' '.join([train_dataset.source_index[i] for i in test_dataset.source_vector[n][:np.argmax(test_dataset.source_vector[n], 0)+1]]))\r\n",
        "\r\n",
        "result, attention = predict(test_dataset.source_vector[n][:np.argmax(test_dataset.source_vector[n], 0)+1])\r\n",
        "txt = \"\"\r\n",
        "\r\n",
        "for i in result[1:]:\r\n",
        "  txt+=train_dataset.target_index[i.numpy()]\r\n",
        "print(\"Output:\", txt)\r\n",
        "\r\n",
        "# Attention Weightをプロット\r\n",
        "plot_attention_weights(attention, test_dataset.source_vector[n][:np.argmax(test_dataset.source_vector[n], 0)+1], result[1:], \"decoder_layer4_block2\")"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: <start> 佐藤 で は ない <end>\n",
            "Output: 着られにゃ\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABHgAAACXCAYAAABur+wHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAaKUlEQVR4nO3de7CkdX3n8feXc2aGmWGYkWEEw1UjgsEFLTGGJZEAguImYBl3rUpQCVHUNRpFUcoVN5eKpgxKLC9x8QaSGHXdDSgmUQQVxIAXVEA3Gy8Rog7gjFx0hmFu3/2j+7iH41zOdD/dv9+vz/tVRXG6+8zTn356ns/p853nEpmJJEmSJEmS2rVX6QCSJEmSJEkajgMeSZIkSZKkxjngkSRJkiRJapwDHkmSJEmSpMY54JEkSZIkSWqcAx5JkiRJkqTGOeCRJEmSJElqnAMeSZIkSZKkxjngkSRJkiRJatxEDngi4oyIOKV0DkkLg50jadzsHUnjZOdIbZguHWBETgICuKZ0EEkLgp0jadzsHUnjZOdIDYjMLJ2hUxFxKrAE2Bf4bmbeVDiSpAlm50gaN3tH0jjZOVI7JvEQrd/OzKuAjwK/VzqM1IqIeFVEvKh0jgbZOdKA7J2B2TvSAOycgdk50gBKdM5EDXgi4gnAbQCZuRW4IyKOKJtKql9ELAYOAI6MiCidpxV2jjQ4e2cw9o40GDtnMHaONJhSnTNRAx7gMcBls26/B3hcoSxSS84C/ha4AvjPhbO0xM6RBmfvDMbekQZj5wzGzpEGU6RzJmrAk5kfAU6cdfs+YF25RFIznpiZ38jM64CTS4dphZ0jDcXeGYC9Iw3MzhmAnSMNrEjnTNSAp++CmS8iYhHwlwWzSNWLiDOAT8666/MR4Qef+bNzpD1k7wzN3pH2gJ0zNDtH2gMlO2diLpMeEb8OXA4cGBHfo3cZv+3AtUWDSfW7OzNvnHX7o8ATS4VphZ0jDcXeGYC9Iw3MzhmAnSMNrFjnTOJl0l+cme8unUPSwmDnSBo3e0fSONk5UjsmccBzZWaeWTqH1IKIWAvMLYEE1mfmMQUiNcfOkfaMvTM8e0eaPztneHaONH+lO2diDtGa5aqIOB/4R2AzQGb+a9lIUp0y8xEAEXEh8OnMvCkingE8uWyyptg50h6wdzph70jzZOd0ws6R5ql050ziHjyfnXNXZqYnUZN2ISL+ITOfMev25zPzxF39GfXYOdJg7J3B2TvSnrNzBmfnSHuuVOdM3B48mXlS6QxSg1ZFxOMz8+sR8WQmsBtGxc6RBmbvDMjekQZi5wzIzpEGUqRzJnEPngOA3wVW0DvT+yGZ+YKyqSAinpeZH+x/vQw4GFgDLAG+AZCZ68sl1KhFxN6Zual0jh2JiGOA9wCHAP8GvDAzv1U2VRtq7Rywdxa6mjsH7J1h1No7do5q7h07Z3B2jmpl5/yivUb9BAV8GFgHPBP4Af2NuwJnA0TEscBVwOOAS4DvAu8EDiyWTCMVEcdGxIfpve9VysxbMvPJmflLmXmCH3j2SK2dA/bOgtRC54C9M6Rae+dssHMWohZ6x84Zip2jqtg5OzeJuyYuyczLI+IPMvN9EXEl8PbSoWZ5OvBo4PeBvwOeB1yfmd8smkqdi4gp4A3A8cCfZuYXCkfaqYj4D8A5wMqZ+zLznHKJmlJ754C9syC01Dlg7wyp9t6xcxaIlnrHzhmKnaMq2Dm7N4kDnjsi4nRgbUT8EXB44TwzjomI1wFH0Ntz6lZgb+CXgYyIMzLzaSUDqnPLgHWZeVrpIPNwOXARsLZ0kAbV2jlg7yw0LXUO2DvDqLV37JyFp6XesXMGZ+eoFnbObkziOXiWAvvSO/bylcCHM/Omsql6Z5/PzJMi4iR6x+JdDzwA3AnsD/xrZr6jZEaNVkTsC2zKzM2ls8wVEVdn5qmlc7So1s4Be2ehq7lzwN4ZRq29Y+eo5t6xcwZn56hWds4vmsRz8LwsM+/KzDsy85XAk0oH6puZpB0CfAv4BLARuJveSZfuKpRL4/M04Ib+v4DU5vqI+KOIeMzMf6UDNaTWzgF7Z6GruXPA3hlGrb1j56jm3rFzBmfnqFZ2zhwTswdPRBwIHAlcDLyif/di4G2ZeXSxYH0RsQ54C70CCmAfYBtwKPA9YG1mXlguocYhIg4DPpaZtfxgBHr/AjLnrszMk4uEaUTtnQP2jurtHLB3BlF779g5gnp7x87Zc3aOWmDnPNQknYNnKb0zqT+i//+gN9X9y3KRHuKWzHxTRHyM3mUG/wr4FeA0elmvKRlO3YuIR9LbPXQFcBDwRODJwKtK5tqRzDypdIYG1d45YO8sKC11Dtg7A6q9d+ycBaal3rFzBmLnqCp2zu5NzB48MyLi1zLzxtI55oqIazPz5Ii4DfgScCy9Y1k/Re+yg5+p+Szg2nMR8RLgGOBnwA+BrwFfzMwtRYPtQEQcTu9fZ9YArwUOzcy/K5mpFbV2Dtg7C01LnQP2zjBq7R07Z+FpqXfsnMHZOaqFnTOP553AAc/zgM/QW5FvB/4+My8umwoi4vmZeVlEnJyZ18557NXAeuDSrOgNiYg1mfnj0jlaFxFLMvPB0jl2JSI+DZxPr4ROAa7LzN8om6oNtXYOtNc7dk43WugcsHeGUWvv2DkLVwu9Y+cMzs7pjr3TDTtn5ybxJMvnZOaPgBfQ2z3vOYXzAJCZl/W/vAAgIv73rIdPy8wP1FA+EXFwRJzZL/Lz+rvBNSEiVpXOMFdEnAl8aNbtF0fEBwpG2pnFmfkN6B0cCmwvnKclVXYOtNE7LXcO1Nc7DXUO2DvDqLJ37JzRq61zoKnesXMGZ+cMoeXesXOGUqRzJnHAs7R/Fu07gc1ArZO9FaUD7EQAvwdcBzwKOPjnD0T8SUTsXSrYbBHxqIiYe9K090XEMUUC7UBEHA2cBzyvf3t/epeWPL9krp24MyJeDyyPiD8A/r10oIa00jlQZ+800TlQf+801jlg7wyjld6xc4ZQe+dAc71j5wzOzhlOE71j53SuSOdM1IAnIvaid3zbSfR2hToUeGfRUDu3OiI+HhGfoHe8aC3W0tuFbD3w4sy8ftZjZ2XmpjKx/r+IOIDe390zIuKREXFkRPwWvfL8l7LpHuK7wO8A50bE1cBNwBbgExFxatFks/S3m3OAJfQuK/lI4EVFQzWisc6BOnun+s6BZnqnic4Be2cYjfWOnTOgRjoHGukdO2dwdk4nqu8dO6dbJTtnkq6iBb3p3f/KzNcARMS+wH5lI/VExGXASuCQiPgVYF1mntF/7Oqi4WbJzK0RcSvw5sx8ycz9EfEbwK3lkj3Em+ldAvFgeu/5Fnq7i34FuBo4sVy0hzgSmOofo3xxf7fRFwJTwHvpZa3BzHZzIUB/Sv9c4N1FU7Wh2s6BNnqnkc6BNnqnlc4Be2cY1faOndOpFjoH2ukdO2dwds6QGukdO6dbxTpnovbgAd5P7w2ecTZw2Y6/dbwy8/nAs4GNwDOBgyPi18qm2qnvAP8UEZ+OiD/s71r2XuB1hXMBvXWZmecA387Ml2fmq4DbgRfTu3Rjcf0fMiuBZ8y6+w7gZOAE4Nslcu1EtdtNA6pedw31TtWdA/X3TmOdA5VvO5Wrdt3ZOd2pvXOgud6pdrtpQLXrrqHOgcp7x87pXLHtZqL24MnMeyIiI2IlsAy4JzMfKJ0LICKm+tPbn2TmGyPirfSmjscAl5fON8eSzLwyIr4AnA5sAJ6SmXcVzjXX4llffxZ4K3B/oSxzPRt4OrBPRHyxf2b/C4CX05uO/7eS4WarebupXe3rrqHeaaVzoN7eaaZzoP5tp2Y1rzs7ZyRq7RxoqHdq3m5qV/O6a6hzoJ3esXM6UHK7mcTLpB9G77i8ZcC7M3Nd4UgARMQ/Ag8A/wTMHGcZwHHAlwEy84Nl0j1URLyd3i6EVZ98LmZdZjAiHg4cCHwnMzeWTdYTEVPAjcA1wH8EXpuZ/1w21Y7Vut20oOZ110rvtNI5UHfvtNQ5UPe2U7ta152d072aOwfa6p1at5sW1LruWukcaKd37JzulNpuJu0QLTLzduARwN61lA9AZp4OvAzYn94xjgcD9wKfAe7r/1eLHwCPLh1iZyLibRHxCuDI6F12cDXwx/Smt1VctrEvge9m5gXAWcCbI+KUwpl2qNbtpgU1r7uGeqfqzoFmeqeZzoG6t53a1bru7JzuNNI50FDv1LrdtKDWdddQ50DlvWPndK/UdjNxe/DAz6dlmzNzbeksO9Kfhp6Ymf+zdJYdiYjpzNxaOsfORMSjgScBjwcOolfq24F19M6s/meZub1cwh2LiL1qzDWj9u2mZi2su5p7p/bOgTZ7p/bOgTa2nVrVvu7snOG02DlQf+/Uvt3UrPZ1V3PnQP29Y+eMRontZiIHPJIkSZIkSQvJxB2iJUmSJEmStNBM9IAnIs4tnWF3zNidFnKacbK1sO5ayAht5DRjN1rIWKtW1l0LOc3YjRYyQjs5a9TCujNjN1rICG3kHGfGiR7wANW/2ZixSy3kNONka2HdtZAR2shpxm60kLFWray7FnKasRstZIR2ctaohXVnxm60kBHayOmAR5IkSZIkSfNTzUmWp1Ysz+nVD+t0mdt+uoGpFcs7W97UhuhsWTO2btrA9N7dZQSY2tLte7pl8wYWLe42Y3a/Ktn64Aaml3SX86hDu7+a3Y/Xb2PN6qlOl3nr+jWdLm/7hg3stbzb93vzD3+wLjO7DTqkqRXLc3rNqk6X2XXnsLn7Gfwo3t+913d/UYjN2zayeGpZZ8t78KDuS2fbfRuZWtldxr3u67YbYDQ/Y7Ljv5bbHtjA1NJuM266u77OAVg8vSyXLlrZ2fI2b93I4unu/g4C5F7d986WrRtYNN3te0zHm/SWLRtYtKjjbWW625Cj+Dy2/eHbOl3e1vs2Mt1hL86YWtvt38vNWzawuMP3e9Ome9m8ZQS/KAxp8aqluezAFZ0uc/O9D7B41dLOlvfoJfd3tqwZo/i8/e3b9ul0eZtzE4tj706XuX1Ft8sbze+A3W8mWx78GYuWdPv+bF/c6eLY+sAGpjv+rPPATj7rTHf6LEOYXv0wDrzw5aVj7NLqL1WzunZp+dpuf1iPQjawKq971yWlI8zLEZe/pHSE3frea151e+kMc02vWcVBf/7S0jF2Kf692x/Uo3LEZd0PQ7v2/T9bUjrCbi39dLcfwkdl67Lqfof5BbddfF51nQOwdNFKjj/87NIxdmn7vt394jZKo/iHoq49uKb+Dt/00ntKR5iXFW+qux+/fPM7S0fYoWUHruAp7/kvpWPs0hVHfKp0hHk5/YgTSkfYrY0nHl06wm5tX9RAeQM/Pbj7f3Tr2q1v2/FnHQ/RkiRJkiRJapwDHkmSJEmSpMY54JEkSZIkSWqcAx5JkiRJkqTGOeCRJEmSJElqnAMeSZIkSZKkxjngkSRJkiRJapwDHkmSJEmSpMY54JEkSZIkSWqcAx5JkiRJkqTGOeCRJEmSJElqnAMeSZIkSZKkxjngkSRJkiRJapwDHkmSJEmSpMZ1PuCJiAMj4g9n3b4oIpZ2/TySBHaOpPGycySNk50jaU9Md7WgiNgHuKp/87iIeHZ/+Y/t3wY4LTM3d/WckhYuO0fSONk5ksbJzpE0iM4GPJn5s4h4LbCmf9fpwD3AX/Rv32UBSeqKnSNpnOwcSeNk50gaRGcDnr6pWcvca87tqY6fS5LsHEnjZOdIGic7R9Ie6fIQrbOBs4ElwH70JszbgeNnfc8lmfmhrp5T0sJl50gaJztH0jjZOZIG0eUhWpcCl0bExcD+wFdnPXww8N7M/JfZfyYizgXOBZjab1VXUSQtAEN3zv4rx5RU0iQYpHPgob2z9/S+Y0gqaRJ00TlLD9hnDEkl1aTTQ7Qi4mTgqUACh8x66FDgirnfn5mXAJcALDn84Owyi6TJN1TnPOogO0fSHtnTzoGH9s7KpY+wdyTN27Cds+qoh9s50gLT5SFapwIvpVco+wNfm/Xws7p6HkkCO0fSeNk5ksbJzpE0iL06XNaPgOfSmzBvBjbN+m9rh88jSWDnSBovO0fSONk5kvZYl+fg+SZARAA8Bzh11sOHAu/v6rkkyc6RNE52jqRxsnMkDaLry6STme8A3tH1ciVpR+wcSeNk50gaJztH0p7o8hAtSZIkSZIkFeCAR5IkSZIkqXEOeCRJkiRJkhrngEeSJEmSJKlxDngkSZIkSZIa54BHkiRJkiSpcQ54JEmSJEmSGueAR5IkSZIkqXEOeCRJkiRJkhrngEeSJEmSJKlxDngkSZIkSZIa54BHkiRJkiSpcQ54JEmSJEmSGueAR5IkSZIkqXHTpQPMOGjFPbzxxI+WjrFL/33VGaUjzMu9ty8vHWG3lv2o/tni037p8aUjzM+bSwdo05K74ZfftrV0jF2K7T8tHWFeHvM3/1Y6wm7d/rFfLR1htx58+n2lI8zL1PUrS0do1pZ9p1l76sNLx9ilxfdn6QjzsnR93f0NsGX5VOkIu7X6nDZ+zqw/Zf/SEXZp2211fq49YPH9nHfIp0rH2KXz73xC6Qjzsn3DhtIRdmv59+r/HLH2pNWlI8zLpjZi7lCdbSRJkiRJkqR5c8AjSZIkSZLUOAc8kiRJkiRJjXPAI0mSJEmS1DgHPJIkSZIkSY1zwCNJkiRJktQ4BzySJEmSJEmNc8AjSZIkSZLUOAc8kiRJkiRJjXPAI0mSJEmS1DgHPJIkSZIkSY1zwCNJkiRJktQ4BzySJEmSJEmNc8AjSZIkSZLUOAc8kiRJkiRJjXPAI0mSJEmS1Lh5D3gi4rkR8RcRsWYnjx8eEZd0F03SQmfvSBonO0fSONk5kro2rwFPRCwC/hb4InBlRCzbwbctAX7WYTZJC5i9I2mc7BxJ42TnSBqF+e7B81zghZn5ceCEzNw480BELI+Ii4C/Bv5TRBwVEVdExC0Rcc4IMktaGOwdSeNk50gaJztHUufmO+A5Fvg/AJmZcx7bCPw98AbgXuBPgZcAv9//c5I0CHtH0jjZOZLGyc6R1Ln5DniOArZExKURcU1EfCEiPh4Rq7PnBuCxwN3A72bmWmAVsG5XC42IcyPiKxHxlZ/+ZOtQL0TSxOm8d2Z3zpYtG8bzKiS1YuSfdbY+YO9I+rmRd85967eN/lVIqsp8BzyH0NuN8PWZeUpm/jpwDXAWQEScDJwP3JSZM5OalfQmzkTE4h0tNDMvyczjMvO4FftND/EyJE2gzntnducsWrR8HK9BUjtG/llneqm9I+nnRt45K1dPjfo1SKrMfAc8T8rM/5qZP4ieY4FnAV/ul8trgFcDMevPLAKmIuKVwPM7TS1pIbB3JI2TnSNpnOwcSZ2b14AnMzdExGERcTPwVeA84PzM/GJmbs7MpwP3A7PHxF8AzqS3K+H7O84tacLZO5LGyc6RNE52jqRRmPdxUZl5e0T86qxdBOe6AfjSrO//IXDSkPkkLWD2jqRxsnMkjZOdI6lre3Tim12UD5m5BdgydCJJmsXekTROdo6kcbJzJHVpvufgkSRJkiRJUqUc8EiSJEmSJDXOAY8kSZIkSVLjHPBIkiRJkiQ1zgGPJEmSJElS4xzwSJIkSZIkNc4BjyRJkiRJUuMc8EiSJEmSJDXOAY8kSZIkSVLjHPBIkiRJkiQ1zgGPJEmSJElS4xzwSJIkSZIkNc4BjyRJkiRJUuMc8EiSJEmSJDXOAY8kSZIkSVLjIjNLZwAgIn4M3N7xYvcH1nW8zK6ZsTst5FyoGQ/LzDUdL3Modk71Wshpxm4siM6BkfROC+8vtJHTjN1oISN0n3OhdA608R6bsRstZIQ2co7ts041A55RiIivZOZxpXPsihm700JOM062FtZdCxmhjZxm7EYLGWvVyrprIacZu9FCRmgnZ41aWHdm7EYLGaGNnOPM6CFakiRJkiRJjXPAI0mSJEmS1LhJH/BcUjrAPDSZMSLunHP7qRFx6bBPFBGfi4ijdnD/CyLiHyLihj3JWSEzTrYW1l0LGWFOzgKd87KIuDEi/jki3hURO/p52cK6NONka2XdtZCzWOdExF4R8ZaIuCEibomIN80nY6VayAjt5KxRC+uuyYzj/qwz6/H37eR5WliP0EbOsWWc6AFPZlb/Zptx3m4HLgCmdvYNleTcJTNOthbWXQsZoWzOiDga+G3ghMw8HlgD/Nbc72thXZpxsrWy7lrIWTjjEcCPMvME4AnAUyLiSXO/yfXYnVZy1qiFdWfG+YuIZwKLd/RYLRl3p4Wc48w40QMelRERB0bEJyPi8xHxiYhY3b//BRFxc0R8OSKe079vVURcFRGfjYj3ACt2tMzMvBq4f3yvQlIruu6czPwmcEZmbuvfNQ08MK7XI6luI+ic/5uZb+nf3A/YBnx/TC9HUgNG8ftVRBwAvBr487G9EI3cdOkAatZ+EfG5WbcfBnyt//VFwEcy84MRcSZwIfAK4EHgeHp74VwDfAR4HXBtZr61X1TfGFN+SW0Za+dk5qaIWAW8C/h6f8gsaeEY++ec/vM9Fjg/M3/c7cuR1IBx987/oDfg2dT1C1E5Dng0qJ9k5m/O3IiIpwJn9W8+HjgsIs6ht5fY3f3zVxwOXA1sp1dYAEcDHwbIzPUR8Z2xpJfUmrF2TkQ8DngL8IbMvKnzVyOpdmP/nJOZvxkRDwM+GRF3ZObnOn1Fkmo3tt6JiBcB38rMGyPi8FG8GJXhgEejcAvwgcy8OiKW0Due/BjgTHoT5hXAdbO+9zTg5og4lF4hSdKe6LRzImIN8FfA72TmfWPIL6ktXXfOU4G9M/OqzLwnIm4HVo3hdUhqR9e/Xz0NWBIRVwDLgKMi4qLMfPWIX4dGzAGPRuE84JKIeD29CfOf0Nu98C7g2v7X3++X0xuBv4ne1bHuAG4uE1lSw7runOcAjwSujIiZ+z7Uwkn8JI1F153zdeCdEfEGeuff+Srw8ZG/Ckkt6bR3MvNZM1/39+D5Y4c7kyEys3QGSZIkSZIkDcGraEmSJEmSJDXOAY8kSZIkSVLjHPBIkiRJkiQ1zgGPJEmSJElS4xzwSJIkSZIkNc4BjyRJkiRJUuMc8EiSJEmSJDXOAY8kSZIkSVLj/h+KvilQ9aPT6wAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 1152x576 with 4 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x00sJiqZQb82",
        "outputId": "b70cc731-db91-443c-d040-eeef21f3529a"
      },
      "source": [
        "dataset2 = preprocess_utils.CreateData(\r\n",
        "    corpus_path = './DATA/kesen_ex.tsv',\r\n",
        "    do_shuffle=True,\r\n",
        "    seed_value=123,\r\n",
        "    split_percent=0.95 # 学習データの割合\r\n",
        ")\r\n",
        "\r\n",
        "train_source2, train_target2, test_source2, test_target2, train_licence2, test_licence2 = dataset2.split_data()\r\n",
        "test_source2[-1]= \"イクコトガデキネエ\"\r\n",
        "test_target2[-1]= \"\"\r\n",
        "test_licence2[-1]= \"大船渡市22\"\r\n",
        "print('**** Test data example ****')\r\n",
        "print('Source Example： ', test_source2[0])\r\n",
        "print('Target Example： ', test_target2[0])\r\n",
        "print('Licence： ', test_licence2[-1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "**** Test data example ****\n",
            "Source Example：  ゼンゼン\n",
            "Target Example：  マッタグ\n",
            "Licence：  大船渡市22\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 379
        },
        "id": "xp1RpU6HRhvZ",
        "outputId": "7b1af312-7d01-47e8-89de-6c15f5288eaf"
      },
      "source": [
        "n = 3# 任意のテストデータを指定\r\n",
        "yyy = (test_dataset.source_vector[n][:np.argmax(test_dataset.source_vector[n], 0)+1])\r\n",
        "\r\n",
        "#yyy[1] = '着ることができる'\r\n",
        "\r\n",
        "result, attention = predict(yyy)\r\n",
        "txt = \"\"\r\n",
        "\r\n",
        "for i in result[1:]:\r\n",
        "  txt+=train_dataset.target_index[i.numpy()]\r\n",
        "print(\"Output:\", txt)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-29-4ee4a309f144>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m#yyy[1] = '着ることができる'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattention\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myyy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0mtxt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-15-771b0cd34c8d>\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(input_vec)\u001b[0m\n\u001b[1;32m      8\u001b[0m   \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_dataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_length\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     enc_padding_mask, combined_mask, dec_padding_mask = model.create_masks(\n\u001b[0;32m---> 10\u001b[0;31m         encoder_input, output)\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/drive/My Drive/Colab Notebooks/Transformer/model.py\u001b[0m in \u001b[0;36mcreate_masks\u001b[0;34m(inp, tar)\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcreate_masks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m   \u001b[0menc_padding_mask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_padding_mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m   \u001b[0mdec_padding_mask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_padding_mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/drive/My Drive/Colab Notebooks/Transformer/model.py\u001b[0m in \u001b[0;36mcreate_padding_mask\u001b[0;34m(seq)\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;31m# Masking\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcreate_padding_mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m   \u001b[0mseq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mequal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mseq\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnewaxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnewaxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/dispatch.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    199\u001b[0m     \u001b[0;34m\"\"\"Call target, and fall back on dispatchers if there is a TypeError.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 201\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    202\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mTypeError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m       \u001b[0;31m# Note: convert_to_eager_tensor currently raises a ValueError, not a\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py\u001b[0m in \u001b[0;36mequal\u001b[0;34m(x, y, name)\u001b[0m\n\u001b[1;32m   1677\u001b[0m     \u001b[0;31m`\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mInvalidArgumentError\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mIf\u001b[0m \u001b[0mshapes\u001b[0m \u001b[0mof\u001b[0m \u001b[0marguments\u001b[0m \u001b[0mare\u001b[0m \u001b[0mincompatible\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1678\u001b[0m   \"\"\"\n\u001b[0;32m-> 1679\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mgen_math_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mequal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1680\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1681\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gen_math_ops.py\u001b[0m in \u001b[0;36mequal\u001b[0;34m(x, y, incompatible_shape_error, name)\u001b[0m\n\u001b[1;32m   3159\u001b[0m       _result = pywrap_tfe.TFE_Py_FastPathExecute(\n\u001b[1;32m   3160\u001b[0m         \u001b[0m_ctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Equal\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"incompatible_shape_error\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3161\u001b[0;31m         incompatible_shape_error)\n\u001b[0m\u001b[1;32m   3162\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3163\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0m_core\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: Cannot convert 0 to EagerTensor of dtype string"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xejq7QGoUvpH"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}