6.5. 循环神经网络的简洁实现.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "6.5. 循环神经网络的简洁实现.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/chongzicbo/Dive-into-Deep-Learning-tf.keras/blob/master/6.5.%20%E5%BE%AA%E7%8E%AF%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%9A%84%E7%AE%80%E6%B4%81%E5%AE%9E%E7%8E%B0.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0V9Z4fTOmRZb",
        "colab_type": "text"
      },
      "source": [
        "##6.5. 循环神经网络的简洁实现\n",
        "本节将使用tf.keras来更简洁地实现基于循环神经网络的语言模型。首先，我们读取周杰伦专辑歌词数据集。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K9VCXtBCONvA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%matplotlib inline\n",
        "import math\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "from IPython import display\n",
        "import matplotlib.pyplot as plt\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import losses\n",
        "from tensorflow.data import Dataset\n",
        "import time\n",
        "import random\n",
        "import zipfile"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yjZQz4M4O53R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tf.enable_eager_execution()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RtjKsiEpO7ul",
        "colab_type": "code",
        "outputId": "b86b7c44-4232-46d4-8a65-770d76bbdc80",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "def load_data_jay_lyrics():\n",
        "  from google.colab import drive\n",
        "  drive.mount('/content/drive')\n",
        "  with zipfile.ZipFile('/content/drive/My Drive/data/d2l-zh-tensoflow/jaychou_lyrics.txt.zip')as zin:\n",
        "    with zin.open('jaychou_lyrics.txt') as f:\n",
        "      corpus_chars=f.read().decode('utf-8')\n",
        "  corpus_chars=corpus_chars.replace('\\n',' ').replace('\\r',' ')\n",
        "  corpus_chars=corpus_chars[0:10000]\n",
        "  idx_to_char=list(set(corpus_chars))\n",
        "  char_to_idx=dict([(char,i) for i,char in enumerate(idx_to_char)])\n",
        "  vocab_size=len(char_to_idx)\n",
        "  corpus_indices=[char_to_idx[char] for char in corpus_chars]\n",
        "  return corpus_indices,char_to_idx,idx_to_char,vocab_size\n",
        "\n",
        "(corpus_indices,char_to_idx,idx_to_char,vocab_size)=load_data_jay_lyrics() "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ujEaMDnDoE29",
        "colab_type": "code",
        "outputId": "75e50424-5ecc-42a1-bd3a-97bbb32fd920",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        }
      },
      "source": [
        "seq_length=100\n",
        "examples_per_epoch=len(corpus_indices) //seq_length\n",
        "char_dataset=Dataset.from_tensor_slices(np.array(corpus_indices))\n",
        "sequences=char_dataset.batch(seq_length+1,drop_remainder=True)\n",
        "def split_input_target(chunk):\n",
        "  input_text=chunk[:-1]\n",
        "  target_text=chunk[1:]\n",
        "  return input_text,target_text\n",
        "dataset=sequences.map(split_input_target)\n",
        "BATCH_SIZE=64\n",
        "setps_per_epoch=examples_per_epoch // BATCH_SIZE\n",
        "BUFFER_SIZE=10000\n",
        "dataset=dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE,drop_remainder=True)\n",
        "for x in dataset:\n",
        "  print(x)\n",
        "  break"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(<tf.Tensor: id=998933, shape=(64, 100), dtype=int64, numpy=\n",
            "array([[ 734,  921,  813, ...,  257,  986,  807],\n",
            "       [ 734,  595,  418, ...,   32,  874,  807],\n",
            "       [ 450,  794,   12, ...,  572,  807,  932],\n",
            "       ...,\n",
            "       [ 869,   89,  212, ...,  986, 1005,  807],\n",
            "       [ 203,  652,  841, ...,  872,  807,  654],\n",
            "       [ 104,  257,  807, ...,  230,   21,  257]])>, <tf.Tensor: id=998934, shape=(64, 100), dtype=int64, numpy=\n",
            "array([[ 921,  813,  807, ...,  986,  807,  257],\n",
            "       [ 595,  418,  806, ...,  874,  807,  631],\n",
            "       [ 794,   12,  526, ...,  807,  932,  558],\n",
            "       ...,\n",
            "       [  89,  212,  943, ..., 1005,  807,  388],\n",
            "       [ 652,  841,  807, ...,  807,  654,  433],\n",
            "       [ 257,  807,  583, ...,   21,  257,  807]])>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zoertaYXmhAd",
        "colab_type": "text"
      },
      "source": [
        "###6.5.1. 定义模型\n",
        "keras的layers模块提供了循环神经网络的实现。下面构造一个含单隐藏层、隐藏单元个数为256的循环神经网络层，并编译、训练。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B6wKgW4ZPLhy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "num_hiddens=256\n",
        "embedding_dim=256\n",
        "net=keras.Sequential()\n",
        "net.add(keras.layers.Embedding(input_dim=vocab_size,output_dim=vocab_size,batch_input_shape=(BATCH_SIZE,seq_length)))\n",
        "net.add(keras.layers.SimpleRNN(num_hiddens,unroll=True,return_sequences=True,stateful=True))\n",
        "net.add(keras.layers.Dense(vocab_size,activation='softmax'))\n",
        "net.compile(optimizer=keras.optimizers.Adam(),loss=losses.SparseCategoricalCrossentropy(),metrics=['acc'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l4Hw3-qVZKIW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net.fit_generator(dataset.repeat(),steps_per_epoch=setps_per_epoch,epochs=500)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8uFzIL5YEr5H",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "new_net=keras.Sequential()\n",
        "new_net.add(keras.layers.Embedding(input_dim=vocab_size,output_dim=vocab_size,batch_input_shape=(1,1)))\n",
        "new_net.add(keras.layers.SimpleRNN(num_hiddens,unroll=True,return_sequences=True,stateful=True))\n",
        "new_net.add(keras.layers.Dense(vocab_size,activation='softmax'))\n",
        "new_net.set_weights(net.get_weights())\n",
        "new_net.compile(optimizer=keras.optimizers.Adam(),loss=losses.SparseCategoricalCrossentropy(),metrics=['acc'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2EDgu4OAn3m5",
        "colab_type": "text"
      },
      "source": [
        "### 6.5.2. 文本生成"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jtU-FZcCJXQX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text_generated=['分','开']\n",
        "for i in range(10):\n",
        "  id=char_to_idx[text_generated[-1]]\n",
        "  char=idx_to_char[tf.argmax(new_net.predict(tf.constant(value=[id]))[0],axis=-1).numpy()[0]]\n",
        "  text_generated.append(char)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aEHnOCLVJla4",
        "colab_type": "code",
        "outputId": "1a55b7be-0b5c-4f41-a98d-81058e16f6a1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "idx_to_char[tf.argmax(new_net.predict(tf.constant(value=[1]))[0],axis=-1).numpy()[0]]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'底'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 82
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "52apQU_vL_WI",
        "colab_type": "code",
        "outputId": "9e0b42ac-2445-42d7-f22a-08abb68cb9f1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "text_generated"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['分', '开', '不', '了', '口', ' ', '周', '杰', '伦', ' ', '才', '离']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 83
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l3Lw66zGNviP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}