notebooks/2019cnps_addtion_rnn.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "2019cnps_addtion_rnn.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ShinAsakawa/2019cnps/blob/master/notebooks/2019cnps_addtion_rnn.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# エンコーダ・デコーダモデル (seq2seq) による 足し算のデモ\n",
        "\n",
        "\n",
        "* 入力: \"535+61\"\n",
        "* 出力 \"596\"\n",
        "* 埋め草文字として空白の繰り返しを用いる\n",
        "\n",
        "* 入力はオプションで反転させることが可能。\n",
        "* 翻訳モデルなどでは，入力系列を反転させることで，性能向上が認められることが知られている。\n",
        "\n",
        "## 文献\n",
        "1. [Learning to Execute](http://arxiv.org/abs/1410.4615) \n",
        "2. [Sequence to Sequence Learning with Neural Networks](http://papers.nips.cc/paper/5346-sequence-to-sequence-learning-with-neural-networks.pdf)\n",
        "\n",
        "## 結果\n",
        "\n",
        "理論的には、ソースとターゲットの間に短期的な依存関係が導入される。\n",
        "以下実行結果\n",
        "\n",
        "* 2 桁 反転 足し算\n",
        "    + 1 層 LSTM 99% 正解率\n",
        "\n",
        "*  3 桁 反転 足し算\n",
        "    + 1 層 LSTM 99% 正解率\n",
        "\n",
        "* 4 桁 反転 足し算。\n",
        "    + 1 層 LSTM 99% 正解率\n",
        "\n",
        "* 5 桁 反転。\n",
        "    + 1 層 LSTM 99% \n"
      ],
      "metadata": {
        "id": "nzfH3NFEy41w"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oyKVWd5s-iuK"
      },
      "source": [
        "from keras.models import Sequential\n",
        "from keras import layers\n",
        "import numpy as np\n",
        "# from six.moves import range"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HLI6yoGe-iuR"
      },
      "source": [
        "class CharacterTable(object):\n",
        "    \"\"\"任意の文字集合が問題として与えられた時に\n",
        "    1. ワンホット 整数表現に符号化\n",
        "    2. ワンホット 整数表現をその文字出力に復号化\n",
        "    3. 出力として得られた確率ベクトルを，対応する文字に復号化\n",
        "    \"\"\"\n",
        "    def __init__(self, chars):\n",
        "        \"\"\"文字表の初期化\n",
        "\n",
        "        引数\n",
        "            chars: 入力信号に現れる全文字集合\n",
        "        \"\"\"\n",
        "        self.chars = sorted(set(chars))\n",
        "        self.char_indices = dict((c, i) for i, c in enumerate(self.chars))\n",
        "        self.indices_char = dict((i, c) for i, c in enumerate(self.chars))\n",
        "\n",
        "    def encode(self, C, num_rows):\n",
        "        \"\"\"文字列 C のワンホット表現\n",
        "\n",
        "        引数\n",
        "            num_rows: ワンホット表現ベクトルを返す行数\n",
        "        \"\"\"\n",
        "        x = np.zeros((num_rows, len(self.chars)))\n",
        "        for i, c in enumerate(C):\n",
        "            x[i, self.char_indices[c]] = 1\n",
        "        return x\n",
        "\n",
        "    def decode(self, x, calc_argmax=True):\n",
        "        if calc_argmax:\n",
        "            x = x.argmax(axis=-1)\n",
        "        return ''.join(self.indices_char[x] for x in x)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gu5w9zuA-iuU"
      },
      "source": [
        "class colors:\n",
        "    ok = '\\033[92m'    # 正解表示用の色を指定\n",
        "    fail = '\\033[91m'  # 失敗表示用の色を指定\n",
        "    close = '\\033[0m'  # 表示色をデフォルトに戻す\n",
        "\n",
        "# 訓練に用いるパラメータ\n",
        "TRAINING_SIZE = 50000\n",
        "DIGITS = 3\n",
        "INVERT = True\n",
        "\n",
        "# 入力の最大長は `数字+数字` で表す\n",
        "MAXLEN = DIGITS + 1 + DIGITS\n",
        "\n",
        "# 入力に用いる文字と埋め草文字(空白)を指定\n",
        "chars = '0123456789+ '\n",
        "ctable = CharacterTable(chars)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t8ivig-G-iue"
      },
      "source": [
        "questions = []\n",
        "expected = []\n",
        "seen = set()\n",
        "print('データの生成...')\n",
        "while len(questions) < TRAINING_SIZE:\n",
        "    f = lambda: int(''.join(np.random.choice(list('0123456789'))\n",
        "                    for i in range(np.random.randint(1, DIGITS + 1))))\n",
        "    a, b = f(), f()\n",
        "\n",
        "    # 既知の足し算問題をスキップ\n",
        "    # 同様に x+Y == Y+x のような問題もスキップ。ソートもする。\n",
        "    key = tuple(sorted((a, b)))\n",
        "    if key in seen:\n",
        "        continue\n",
        "    seen.add(key)\n",
        "\n",
        "    # 常にMAXLENになるようにデータを空白で埋める\n",
        "    q = '{}+{}'.format(a, b)\n",
        "    query = q + ' ' * (MAXLEN - len(q))\n",
        "    ans = str(a + b)\n",
        "    # 回答 `ans` は最大で DIGITS + 1 のサイズ\n",
        "    ans += ' ' * (DIGITS + 1 - len(ans))\n",
        "    if INVERT:\n",
        "        # 例えば '12+345 ' は ' 543+21' となる。\n",
        "        # 埋め草文字の空白注意\n",
        "        query = query[::-1]\n",
        "    questions.append(query)\n",
        "    expected.append(ans)\n",
        "print(f'総問題数: {len(questions)}')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qXyg6INT-iuk"
      },
      "source": [
        "print('ベクトル化...')\n",
        "x = np.zeros((len(questions), MAXLEN, len(chars)), dtype=np.bool_)\n",
        "y = np.zeros((len(questions), DIGITS + 1, len(chars)), dtype=np.bool_)\n",
        "for i, sentence in enumerate(questions):\n",
        "    x[i] = ctable.encode(sentence, MAXLEN)\n",
        "for i, sentence in enumerate(expected):\n",
        "    y[i] = ctable.encode(sentence, DIGITS + 1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vA9FvRLm-iuo"
      },
      "source": [
        "# x の後半部分はほとんど大きな数字になるためデータ (x, y) をシャッフル\n",
        "indices = np.arange(len(y))\n",
        "np.random.shuffle(indices)\n",
        "x = x[indices]\n",
        "y = y[indices]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bcQjkK6p-ius"
      },
      "source": [
        "# データセットの 10% を検証データセットとして設定。検証データセットでは学習を行わない\n",
        "split_at = len(x) - len(x) // 10\n",
        "(x_train, x_val) = x[:split_at], x[split_at:]\n",
        "(y_train, y_val) = y[:split_at], y[split_at:]\n",
        "\n",
        "print('訓練データセット数:')\n",
        "print(f'入力データサイズ: {x_train.shape}')\n",
        "print(f'教師データサイズ:{y_train.shape}')\n",
        "\n",
        "print('検証データセット数:')\n",
        "print(f'入力データサイズ: {x_val.shape}')\n",
        "print(f'教師データサイズ: {y_val.shape}')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oPXGhQJM-iux"
      },
      "source": [
        "# GRU や SimpleRNN を置き換えて実行してみること\n",
        "RNN = layers.LSTM\n",
        "HIDDEN_SIZE = 128\n",
        "BATCH_SIZE = 128\n",
        "LAYERS = 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_zaTTDf3-ivD"
      },
      "source": [
        "print('モデル作成...')\n",
        "model = Sequential()\n",
        "\n",
        "# RNN を使って入力配列を符号化し，HIDDEN_SIZE の出力を生成\n",
        "# 注：入力系列が可変長の場合には `input_shape=(None, num_feature)` を使用\n",
        "model.add(RNN(HIDDEN_SIZE, input_shape=(MAXLEN, len(chars))))\n",
        "\n",
        "# 復号化器 RNN の入力として，最終時刻の隠れ層の状態を繰り返し与える。\n",
        "# RNN の最終時刻の隠れ層の状態を，各時間ステップで繰り返し提供。\n",
        "# すなわち，DIGITS + 1 回繰り返す。\n",
        "# 例えば DIGITS=3 の場合，最大出力は999+999=1998 となる。\n",
        "model.add(layers.RepeatVector(DIGITS + 1))\n",
        "\n",
        "# 復号化器 RNN は，複数層を重ねることも，単層にすることも可能\n",
        "for _ in range(LAYERS):\n",
        "    # `return_sequences=True` にすると，最後の出力だけでなく，これまでの全出力を (num_samples, timesteps, output_dim) という形で返す。\n",
        "    # これは，下記  `TimeDistributed` が最初の次元を `timesteps` と想定しているため，必要となる。\n",
        "    model.add(RNN(HIDDEN_SIZE, return_sequences=True))\n",
        "\n",
        "# 入力の各時間スライスに全結合層を適用する。\n",
        "# 出力系列の各時刻に対して，どの文字を選択するかを決定するため\n",
        "model.add(layers.TimeDistributed(layers.Dense(len(chars))))\n",
        "model.add(layers.Activation('softmax'))\n",
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer='adam',\n",
        "              metrics=['accuracy'])\n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y3vy6Lvk-ivI"
      },
      "source": [
        "# モデルをエポックごとに学習させ，検証用データセットに対する予測値を表示\n",
        "epochs = 30\n",
        "n_sample = 10\n",
        "for epoch in range(epochs):\n",
        "    print()\n",
        "    print('-' * 78)\n",
        "    print(f'エポック数: {epoch+1}')\n",
        "    model.fit(x_train, y_train,\n",
        "              batch_size=BATCH_SIZE,\n",
        "              epochs=1, validation_data=(x_val, y_val))\n",
        "    \n",
        "    # 検証データセットから n_sample 個の事例をランダムサンプリングして結果を表示\n",
        "    for i in range(n_sample):\n",
        "        ind = np.random.randint(0, len(x_val))\n",
        "        rowx, rowy = x_val[np.array([ind])], y_val[np.array([ind])]\n",
        "\n",
        "        preds = np.argmax(model.predict(rowx), axis=-1)\n",
        "        q = ctable.decode(rowx[0])\n",
        "        correct = ctable.decode(rowy[0])\n",
        "        guess = ctable.decode(preds[0], calc_argmax=False)\n",
        "        print('問:', q[::-1] if INVERT else q)\n",
        "        print('答:', correct)\n",
        "        if correct == guess:\n",
        "            print(colors.ok + '正' + colors.close, end=\" \")\n",
        "        else:\n",
        "            print(colors.fail + '誤' + colors.close, end=\" \")\n",
        "        print(guess)\n",
        "        print('---')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}