261_使用注意力机制的seq2seq.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. 使用注意力机制的seq2seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from d2l import torch as d2l\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 带有注意力机制的解码器基本接口\n",
    "class AttentionDecoder(d2l.Decoder):\n",
    "    \"\"\"带有注意力机制的解码器基本接口\"\"\"\n",
    "    def __init__(self, **kwargs):\n",
    "        super(AttentionDecoder, self).__init__(**kwargs)\n",
    "        \n",
    "    @property\n",
    "    def attention_weight(self):\n",
    "        raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 实现带有Bahdanau注意力的循环神经网络解码器\n",
    "class Seq2SeqAttentionDecoder(AttentionDecoder):\n",
    "    def __init__(self, vocab_size, embed_size, num_hiddens, num_layers, dropout=0, **kwargs):\n",
    "        # 调用父类AttentionDecoder的构造函数进行初始化\n",
    "        super(Seq2SeqAttentionDecoder, self).__init__(**kwargs)\n",
    "        # 创建一个加性注意力机制的实例，用于计算注意力权重\n",
    "        self.attention = d2l.AdditiveAttention(num_hiddens, num_hiddens, num_hiddens, dropout)  \n",
    "        # 创建一个嵌入层，用于将输入的整数序列进行嵌入表示\n",
    "        self.embedding = nn.Embedding(vocab_size, embed_size)\n",
    "        # 创建一个GRU层，用于实现循环神经网络的计算\n",
    "        self.rnn = nn.GRU(embed_size + num_hiddens, num_hiddens, num_layers, dropout=dropout)  \n",
    "        # 创建一个线性层，将隐藏状态映射到词汇表大小的输出\n",
    "        self.dense = nn.Linear(num_hiddens, vocab_size)\n",
    "        \n",
    "    def init_state(self, enc_outputs, enc_valid_lens, *args):\n",
    "        # 将编码器的输出解包为outputs和hidden_state\n",
    "        outputs, hidden_state = enc_outputs\n",
    "        # 对outputs进行维度变换，将batch维和时间步维交换，保持与解码器输入的一致性\n",
    "        # 返回初始化的解码器隐藏状态\n",
    "        return (outputs.permute(1, 0, 2), hidden_state, enc_valid_lens)\n",
    "    \n",
    "    def forward(self, X, state):\n",
    "        # 解析输入的状态信息，包括编码器的输出、隐藏状态和有效长度\n",
    "        enc_outputs, hidden_state, enc_valid_lens = state\n",
    "        # 对输入序列进行嵌入表示，并进行维度变换，将batch维和时间步维交换\n",
    "        X = self.embedding(X).permute(1, 0, 2)\n",
    "        # 初始化输出列表和注意力权重列表\n",
    "        outputs, self._attention_weights = [], []\n",
    "        for x in X:\n",
    "            # 获取当前时间步的查询向量，将隐藏状态的最后一个时间步的特征进行维度扩展\n",
    "            query = torch.unsqueeze(hidden_state[-1], dim=1)\n",
    "            # 计算注意力上下文向量，通过注意力机制对编码器的输出进行加权求和\n",
    "            context = self.attention(query, enc_outputs, enc_outputs, enc_valid_lens)  \n",
    "            # 将注意力上下文向量与当前时间步的输入进行拼接，用于输入到循环神经网络\n",
    "            x = torch.cat((context, torch.unsqueeze(x, dim=1)), dim=-1)\n",
    "            # 执行循环神经网络的前向计算，得到输出和更新后的隐藏状态\n",
    "            out, hidden_state = self.rnn(x.permute(1,0,2),hidden_state)\n",
    "            # 将输出和注意力权重添加到对应的列表中\n",
    "            outputs.append(out)\n",
    "            self._attention_weights.append(self.attention.attention_weights)\n",
    "        # 将输出列表中的结果进行拼接，并通过线性层进行映射\n",
    "        outputs = self.dense(torch.cat(outputs, dim=0))\n",
    "        # 对输出进行维度变换，将batch维和时间步维交换，并返回更新后的状态信息\n",
    "        return outputs.permute(1, 0, 2), [enc_outputs, hidden_state, enc_valid_lens]  \n",
    "    \n",
    "    @property\n",
    "    def attention_weights(self):\n",
    "        # 返回注意力权重\n",
    "        return self._attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([4, 7, 10]), 3, torch.Size([4, 7, 16]), 2, torch.Size([4, 16]))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 测试Bahdanau注意力解码器\n",
    "# 创建编码器和解码器的实例\n",
    "encoder = d2l.Seq2SeqEncoder(vocab_size = 10, embed_size=8, num_hiddens=16, num_layers=2)  \n",
    "encoder.eval()\n",
    "decoder = Seq2SeqAttentionDecoder(vocab_size=10, embed_size=8, num_hiddens=16,num_layers=2)  \n",
    "decoder.eval()\n",
    "# 创建输入序列X\n",
    "X = torch.zeros((4,7), dtype=torch.long)\n",
    "# 初始化解码器的状态\n",
    "state = decoder.init_state(encoder(X), None)\n",
    "# 执行解码器的前向计算\n",
    "output, state = decoder(X, state)\n",
    "# 输出结果的形状以及状态信息的长度和形状\n",
    "output.shape, len(state), state[0].shape, len(state[1]), state[1][0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_data_nmt():\n",
    "    \"\"\"载入 “英语-法语” 数据集 \"\"\"\n",
    "    # 下载并解压数据集\n",
    "    data_dir = d2l.download_extract('fra-eng')\n",
    "    # 打开数据文件，使用utf-8编码读取文件内容\n",
    "    with open(os.path.join(data_dir, 'fra.txt'), 'r', encoding='utf-8') as f:\n",
    "        # 返回读取的文件内容\n",
    "        return f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_nmt(text):\n",
    "    \"\"\"预处理 “英语-法语” 数据集\"\"\"\n",
    "    def no_space(char, prev_char):\n",
    "        # 检查是否需要在字符之前添加空格\n",
    "        return char in set(',.!?') and prev_char != ''\n",
    "    # 替换特殊字符为空格并转换为小写\n",
    "    text = text.replace('\\u202f', ' ').replace('\\xa0',' ').lower()\n",
    "    # 根据规则在字符之前添加空格\n",
    "    out = [\n",
    "        ' ' + char if i > 0 and no_space(char, text[i - 1]) else char\n",
    "        for i, char in enumerate(text)]\n",
    "    # 将处理后的字符列表连接成字符串并返回\n",
    "    return ''.join(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_nmt(text, num_examples=None):\n",
    "    \"\"\"词元化 “英语-法语” 数据数据集 \"\"\"\n",
    "    # 初始化源语言和目标语言的列表\n",
    "    source, target = [], []\n",
    "    # 对每行文本进行遍历\n",
    "    for i, line in enumerate(text.split('\\n')):\n",
    "        # 如果指定了num_examples，并且超过了指定数量，则退出循环\n",
    "        if num_examples and i > num_examples:\n",
    "            break\n",
    "        # 将每行文本按制表符分割成源语言和目标语言的部分\n",
    "        parts = line.split('\\t')\n",
    "        # 如果分割后有两个部分，说明包含了源语言和目标语言的内容\n",
    "        if len(parts) == 2:\n",
    "            # 将源语言部分以空格为分隔符进行词元化，存储到源语言列表中\n",
    "            source.append(parts[0].split(' '))\n",
    "            # 将目标语言部分以空格为分隔符进行词元化，存储到目标语言列表中\n",
    "            target.append(parts[1].split(' '))\n",
    "    # 返回词元化后的源语言列表和目标语言列表\n",
    "    return source, target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def truncate_pad(line, num_steps, padding_token):\n",
    "    \"\"\"截断或填充文本序列\"\"\"\n",
    "    # 如果文本序列的长度超过了指定的步数\n",
    "    if len(line) > num_steps:\n",
    "        # 截断文本序列，保留前num_steps个元素\n",
    "        return line[:num_steps]\n",
    "    # 在文本序列末尾填充padding_token，使其长度达到num_steps\n",
    "    return line + [padding_token] * (num_steps - len(line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_array_nmt(lines, vocab, num_steps):\n",
    "    \"\"\"将机器翻译的文本序列转换成小批量\"\"\"\n",
    "    # 将文本序列中的每个词根据词汇表转换为对应的索引\n",
    "    lines = [vocab[l] for l in lines]\n",
    "    # 在每个文本序列末尾添加<eos>表示句子结束的索引\n",
    "    lines = [l + [vocab['<eos>']] for l in lines]\n",
    "    # 将每个文本序列截断或填充为固定长度，并转换为tensor形式\n",
    "    array = torch.tensor([ truncate_pad(l, num_steps, vocab['<pad>']) for l in lines ])\n",
    "    # 计算有效长度，即非填充部分的长度\n",
    "    valid_len = (array != vocab['<pad>']).type(torch.int32).sum(1)\n",
    "    # 返回转换后的tensor形式的文本序列和有效长度\n",
    "    return array, valid_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data_nmt(batch_size, num_steps, num_examples=600):\n",
    "    \"\"\"返回翻译数据集的迭代器和词汇表\"\"\"\n",
    "    # 载入并预处理文本数据集\n",
    "    text = preprocess_nmt(read_data_nmt())\n",
    "    # 将文本数据集进行词元化\n",
    "    source, target = tokenize_nmt(text, num_examples)\n",
    "    # 构建源语言的词汇表对象\n",
    "    src_vocab = d2l.Vocab(source, min_freq=2,\n",
    "                         reserved_tokens=['<pad>','<bos>','<eos>'])\n",
    "    # 构建目标语言的词汇表对象\n",
    "    tgt_vocab = d2l.Vocab(target, min_freq=2,\n",
    "                         reserved_tokens=['<pad>','<bos>','<eos>'])\n",
    "    # 将源语言文本序列转换为小批量数据，并获取有效长度\n",
    "    src_array, src_valid_len = build_array_nmt(source, src_vocab, num_steps)  \n",
    "    # 将目标语言文本序列转换为小批量数据，并获取有效长度\n",
    "    tgt_array, tgt_valid_len = build_array_nmt(target, tgt_vocab, num_steps)\n",
    "    # 构建数据集的数组\n",
    "    data_arrays = (src_array, src_valid_len, tgt_array, tgt_valid_len)\n",
    "    # 构建数据迭代器\n",
    "    data_iter = d2l.load_array(data_arrays, batch_size)\n",
    "    # 返回数据迭代器和词汇表对象\n",
    "    return data_iter, src_vocab, tgt_vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss 0.018, 7276.1 tokens/sec on cuda:0\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\r\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n",
       "  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n",
       "<!-- Created with matplotlib (https://matplotlib.org/) -->\r\n",
       "<svg height=\"180.65625pt\" version=\"1.1\" viewBox=\"0 0 262.1875 180.65625\" width=\"262.1875pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n",
       " <metadata>\r\n",
       "  <rdf:RDF xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\r\n",
       "   <cc:Work>\r\n",
       "    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\r\n",
       "    <dc:date>2022-08-03T16:04:48.820464</dc:date>\r\n",
       "    <dc:format>image/svg+xml</dc:format>\r\n",
       "    <dc:creator>\r\n",
       "     <cc:Agent>\r\n",
       "      <dc:title>Matplotlib v3.3.4, https://matplotlib.org/</dc:title>\r\n",
       "     </cc:Agent>\r\n",
       "    </dc:creator>\r\n",
       "   </cc:Work>\r\n",
       "  </rdf:RDF>\r\n",
       " </metadata>\r\n",
       " <defs>\r\n",
       "  <style type=\"text/css\">*{stroke-linecap:butt;stroke-linejoin:round;}</style>\r\n",
       " </defs>\r\n",
       " <g id=\"figure_1\">\r\n",
       "  <g id=\"patch_1\">\r\n",
       "   <path d=\"M 0 180.65625 \r\n",
       "L 262.1875 180.65625 \r\n",
       "L 262.1875 0 \r\n",
       "L 0 0 \r\n",
       "z\r\n",
       "\" style=\"fill:none;\"/>\r\n",
       "  </g>\r\n",
       "  <g id=\"axes_1\">\r\n",
       "   <g id=\"patch_2\">\r\n",
       "    <path d=\"M 50.14375 143.1 \r\n",
       "L 245.44375 143.1 \r\n",
       "L 245.44375 7.2 \r\n",
       "L 50.14375 7.2 \r\n",
       "z\r\n",
       "\" style=\"fill:#ffffff;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"matplotlib.axis_1\">\r\n",
       "    <g id=\"xtick_1\">\r\n",
       "     <g id=\"line2d_1\">\r\n",
       "      <path clip-path=\"url(#p3f4b1bc450)\" d=\"M 82.69375 143.1 \r\n",
       "L 82.69375 7.2 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_2\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 0 0 \r\n",
       "L 0 3.5 \r\n",
       "\" id=\"md9aa9d4f72\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n",
       "      </defs>\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"82.69375\" xlink:href=\"#md9aa9d4f72\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_1\">\r\n",
       "      <!-- 50 -->\r\n",
       "      <g transform=\"translate(76.33125 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 10.796875 72.90625 \r\n",
       "L 49.515625 72.90625 \r\n",
       "L 49.515625 64.59375 \r\n",
       "L 19.828125 64.59375 \r\n",
       "L 19.828125 46.734375 \r\n",
       "Q 21.96875 47.46875 24.109375 47.828125 \r\n",
       "Q 26.265625 48.1875 28.421875 48.1875 \r\n",
       "Q 40.625 48.1875 47.75 41.5 \r\n",
       "Q 54.890625 34.8125 54.890625 23.390625 \r\n",
       "Q 54.890625 11.625 47.5625 5.09375 \r\n",
       "Q 40.234375 -1.421875 26.90625 -1.421875 \r\n",
       "Q 22.3125 -1.421875 17.546875 -0.640625 \r\n",
       "Q 12.796875 0.140625 7.71875 1.703125 \r\n",
       "L 7.71875 11.625 \r\n",
       "Q 12.109375 9.234375 16.796875 8.0625 \r\n",
       "Q 21.484375 6.890625 26.703125 6.890625 \r\n",
       "Q 35.15625 6.890625 40.078125 11.328125 \r\n",
       "Q 45.015625 15.765625 45.015625 23.390625 \r\n",
       "Q 45.015625 31 40.078125 35.4375 \r\n",
       "Q 35.15625 39.890625 26.703125 39.890625 \r\n",
       "Q 22.75 39.890625 18.8125 39.015625 \r\n",
       "Q 14.890625 38.140625 10.796875 36.28125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-53\"/>\r\n",
       "        <path d=\"M 31.78125 66.40625 \r\n",
       "Q 24.171875 66.40625 20.328125 58.90625 \r\n",
       "Q 16.5 51.421875 16.5 36.375 \r\n",
       "Q 16.5 21.390625 20.328125 13.890625 \r\n",
       "Q 24.171875 6.390625 31.78125 6.390625 \r\n",
       "Q 39.453125 6.390625 43.28125 13.890625 \r\n",
       "Q 47.125 21.390625 47.125 36.375 \r\n",
       "Q 47.125 51.421875 43.28125 58.90625 \r\n",
       "Q 39.453125 66.40625 31.78125 66.40625 \r\n",
       "z\r\n",
       "M 31.78125 74.21875 \r\n",
       "Q 44.046875 74.21875 50.515625 64.515625 \r\n",
       "Q 56.984375 54.828125 56.984375 36.375 \r\n",
       "Q 56.984375 17.96875 50.515625 8.265625 \r\n",
       "Q 44.046875 -1.421875 31.78125 -1.421875 \r\n",
       "Q 19.53125 -1.421875 13.0625 8.265625 \r\n",
       "Q 6.59375 17.96875 6.59375 36.375 \r\n",
       "Q 6.59375 54.828125 13.0625 64.515625 \r\n",
       "Q 19.53125 74.21875 31.78125 74.21875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-48\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-53\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_2\">\r\n",
       "     <g id=\"line2d_3\">\r\n",
       "      <path clip-path=\"url(#p3f4b1bc450)\" d=\"M 123.38125 143.1 \r\n",
       "L 123.38125 7.2 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_4\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"123.38125\" xlink:href=\"#md9aa9d4f72\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_2\">\r\n",
       "      <!-- 100 -->\r\n",
       "      <g transform=\"translate(113.8375 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 12.40625 8.296875 \r\n",
       "L 28.515625 8.296875 \r\n",
       "L 28.515625 63.921875 \r\n",
       "L 10.984375 60.40625 \r\n",
       "L 10.984375 69.390625 \r\n",
       "L 28.421875 72.90625 \r\n",
       "L 38.28125 72.90625 \r\n",
       "L 38.28125 8.296875 \r\n",
       "L 54.390625 8.296875 \r\n",
       "L 54.390625 0 \r\n",
       "L 12.40625 0 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-49\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-49\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_3\">\r\n",
       "     <g id=\"line2d_5\">\r\n",
       "      <path clip-path=\"url(#p3f4b1bc450)\" d=\"M 164.06875 143.1 \r\n",
       "L 164.06875 7.2 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_6\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"164.06875\" xlink:href=\"#md9aa9d4f72\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_3\">\r\n",
       "      <!-- 150 -->\r\n",
       "      <g transform=\"translate(154.525 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-49\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\r\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_4\">\r\n",
       "     <g id=\"line2d_7\">\r\n",
       "      <path clip-path=\"url(#p3f4b1bc450)\" d=\"M 204.75625 143.1 \r\n",
       "L 204.75625 7.2 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_8\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"204.75625\" xlink:href=\"#md9aa9d4f72\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_4\">\r\n",
       "      <!-- 200 -->\r\n",
       "      <g transform=\"translate(195.2125 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 19.1875 8.296875 \r\n",
       "L 53.609375 8.296875 \r\n",
       "L 53.609375 0 \r\n",
       "L 7.328125 0 \r\n",
       "L 7.328125 8.296875 \r\n",
       "Q 12.9375 14.109375 22.625 23.890625 \r\n",
       "Q 32.328125 33.6875 34.8125 36.53125 \r\n",
       "Q 39.546875 41.84375 41.421875 45.53125 \r\n",
       "Q 43.3125 49.21875 43.3125 52.78125 \r\n",
       "Q 43.3125 58.59375 39.234375 62.25 \r\n",
       "Q 35.15625 65.921875 28.609375 65.921875 \r\n",
       "Q 23.96875 65.921875 18.8125 64.3125 \r\n",
       "Q 13.671875 62.703125 7.8125 59.421875 \r\n",
       "L 7.8125 69.390625 \r\n",
       "Q 13.765625 71.78125 18.9375 73 \r\n",
       "Q 24.125 74.21875 28.421875 74.21875 \r\n",
       "Q 39.75 74.21875 46.484375 68.546875 \r\n",
       "Q 53.21875 62.890625 53.21875 53.421875 \r\n",
       "Q 53.21875 48.921875 51.53125 44.890625 \r\n",
       "Q 49.859375 40.875 45.40625 35.40625 \r\n",
       "Q 44.1875 33.984375 37.640625 27.21875 \r\n",
       "Q 31.109375 20.453125 19.1875 8.296875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-50\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-50\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_5\">\r\n",
       "     <g id=\"line2d_9\">\r\n",
       "      <path clip-path=\"url(#p3f4b1bc450)\" d=\"M 245.44375 143.1 \r\n",
       "L 245.44375 7.2 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_10\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"245.44375\" xlink:href=\"#md9aa9d4f72\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_5\">\r\n",
       "      <!-- 250 -->\r\n",
       "      <g transform=\"translate(235.9 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-50\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\r\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"text_6\">\r\n",
       "     <!-- epoch -->\r\n",
       "     <g transform=\"translate(132.565625 171.376563)scale(0.1 -0.1)\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 56.203125 29.59375 \r\n",
       "L 56.203125 25.203125 \r\n",
       "L 14.890625 25.203125 \r\n",
       "Q 15.484375 15.921875 20.484375 11.0625 \r\n",
       "Q 25.484375 6.203125 34.421875 6.203125 \r\n",
       "Q 39.59375 6.203125 44.453125 7.46875 \r\n",
       "Q 49.3125 8.734375 54.109375 11.28125 \r\n",
       "L 54.109375 2.78125 \r\n",
       "Q 49.265625 0.734375 44.1875 -0.34375 \r\n",
       "Q 39.109375 -1.421875 33.890625 -1.421875 \r\n",
       "Q 20.796875 -1.421875 13.15625 6.1875 \r\n",
       "Q 5.515625 13.8125 5.515625 26.8125 \r\n",
       "Q 5.515625 40.234375 12.765625 48.109375 \r\n",
       "Q 20.015625 56 32.328125 56 \r\n",
       "Q 43.359375 56 49.78125 48.890625 \r\n",
       "Q 56.203125 41.796875 56.203125 29.59375 \r\n",
       "z\r\n",
       "M 47.21875 32.234375 \r\n",
       "Q 47.125 39.59375 43.09375 43.984375 \r\n",
       "Q 39.0625 48.390625 32.421875 48.390625 \r\n",
       "Q 24.90625 48.390625 20.390625 44.140625 \r\n",
       "Q 15.875 39.890625 15.1875 32.171875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-101\"/>\r\n",
       "       <path d=\"M 18.109375 8.203125 \r\n",
       "L 18.109375 -20.796875 \r\n",
       "L 9.078125 -20.796875 \r\n",
       "L 9.078125 54.6875 \r\n",
       "L 18.109375 54.6875 \r\n",
       "L 18.109375 46.390625 \r\n",
       "Q 20.953125 51.265625 25.265625 53.625 \r\n",
       "Q 29.59375 56 35.59375 56 \r\n",
       "Q 45.5625 56 51.78125 48.09375 \r\n",
       "Q 58.015625 40.1875 58.015625 27.296875 \r\n",
       "Q 58.015625 14.40625 51.78125 6.484375 \r\n",
       "Q 45.5625 -1.421875 35.59375 -1.421875 \r\n",
       "Q 29.59375 -1.421875 25.265625 0.953125 \r\n",
       "Q 20.953125 3.328125 18.109375 8.203125 \r\n",
       "z\r\n",
       "M 48.6875 27.296875 \r\n",
       "Q 48.6875 37.203125 44.609375 42.84375 \r\n",
       "Q 40.53125 48.484375 33.40625 48.484375 \r\n",
       "Q 26.265625 48.484375 22.1875 42.84375 \r\n",
       "Q 18.109375 37.203125 18.109375 27.296875 \r\n",
       "Q 18.109375 17.390625 22.1875 11.75 \r\n",
       "Q 26.265625 6.109375 33.40625 6.109375 \r\n",
       "Q 40.53125 6.109375 44.609375 11.75 \r\n",
       "Q 48.6875 17.390625 48.6875 27.296875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-112\"/>\r\n",
       "       <path d=\"M 30.609375 48.390625 \r\n",
       "Q 23.390625 48.390625 19.1875 42.75 \r\n",
       "Q 14.984375 37.109375 14.984375 27.296875 \r\n",
       "Q 14.984375 17.484375 19.15625 11.84375 \r\n",
       "Q 23.34375 6.203125 30.609375 6.203125 \r\n",
       "Q 37.796875 6.203125 41.984375 11.859375 \r\n",
       "Q 46.1875 17.53125 46.1875 27.296875 \r\n",
       "Q 46.1875 37.015625 41.984375 42.703125 \r\n",
       "Q 37.796875 48.390625 30.609375 48.390625 \r\n",
       "z\r\n",
       "M 30.609375 56 \r\n",
       "Q 42.328125 56 49.015625 48.375 \r\n",
       "Q 55.71875 40.765625 55.71875 27.296875 \r\n",
       "Q 55.71875 13.875 49.015625 6.21875 \r\n",
       "Q 42.328125 -1.421875 30.609375 -1.421875 \r\n",
       "Q 18.84375 -1.421875 12.171875 6.21875 \r\n",
       "Q 5.515625 13.875 5.515625 27.296875 \r\n",
       "Q 5.515625 40.765625 12.171875 48.375 \r\n",
       "Q 18.84375 56 30.609375 56 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-111\"/>\r\n",
       "       <path d=\"M 48.78125 52.59375 \r\n",
       "L 48.78125 44.1875 \r\n",
       "Q 44.96875 46.296875 41.140625 47.34375 \r\n",
       "Q 37.3125 48.390625 33.40625 48.390625 \r\n",
       "Q 24.65625 48.390625 19.8125 42.84375 \r\n",
       "Q 14.984375 37.3125 14.984375 27.296875 \r\n",
       "Q 14.984375 17.28125 19.8125 11.734375 \r\n",
       "Q 24.65625 6.203125 33.40625 6.203125 \r\n",
       "Q 37.3125 6.203125 41.140625 7.25 \r\n",
       "Q 44.96875 8.296875 48.78125 10.40625 \r\n",
       "L 48.78125 2.09375 \r\n",
       "Q 45.015625 0.34375 40.984375 -0.53125 \r\n",
       "Q 36.96875 -1.421875 32.421875 -1.421875 \r\n",
       "Q 20.0625 -1.421875 12.78125 6.34375 \r\n",
       "Q 5.515625 14.109375 5.515625 27.296875 \r\n",
       "Q 5.515625 40.671875 12.859375 48.328125 \r\n",
       "Q 20.21875 56 33.015625 56 \r\n",
       "Q 37.15625 56 41.109375 55.140625 \r\n",
       "Q 45.0625 54.296875 48.78125 52.59375 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-99\"/>\r\n",
       "       <path d=\"M 54.890625 33.015625 \r\n",
       "L 54.890625 0 \r\n",
       "L 45.90625 0 \r\n",
       "L 45.90625 32.71875 \r\n",
       "Q 45.90625 40.484375 42.875 44.328125 \r\n",
       "Q 39.84375 48.1875 33.796875 48.1875 \r\n",
       "Q 26.515625 48.1875 22.3125 43.546875 \r\n",
       "Q 18.109375 38.921875 18.109375 30.90625 \r\n",
       "L 18.109375 0 \r\n",
       "L 9.078125 0 \r\n",
       "L 9.078125 75.984375 \r\n",
       "L 18.109375 75.984375 \r\n",
       "L 18.109375 46.1875 \r\n",
       "Q 21.34375 51.125 25.703125 53.5625 \r\n",
       "Q 30.078125 56 35.796875 56 \r\n",
       "Q 45.21875 56 50.046875 50.171875 \r\n",
       "Q 54.890625 44.34375 54.890625 33.015625 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-104\"/>\r\n",
       "      </defs>\r\n",
       "      <use xlink:href=\"#DejaVuSans-101\"/>\r\n",
       "      <use x=\"61.523438\" xlink:href=\"#DejaVuSans-112\"/>\r\n",
       "      <use x=\"125\" xlink:href=\"#DejaVuSans-111\"/>\r\n",
       "      <use x=\"186.181641\" xlink:href=\"#DejaVuSans-99\"/>\r\n",
       "      <use x=\"241.162109\" xlink:href=\"#DejaVuSans-104\"/>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "   </g>\r\n",
       "   <g id=\"matplotlib.axis_2\">\r\n",
       "    <g id=\"ytick_1\">\r\n",
       "     <g id=\"line2d_11\">\r\n",
       "      <path clip-path=\"url(#p3f4b1bc450)\" d=\"M 50.14375 112.413806 \r\n",
       "L 245.44375 112.413806 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_12\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 0 0 \r\n",
       "L -3.5 0 \r\n",
       "\" id=\"mdacda6f47a\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n",
       "      </defs>\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"50.14375\" xlink:href=\"#mdacda6f47a\" y=\"112.413806\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_7\">\r\n",
       "      <!-- 0.05 -->\r\n",
       "      <g transform=\"translate(20.878125 116.213025)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 10.6875 12.40625 \r\n",
       "L 21 12.40625 \r\n",
       "L 21 0 \r\n",
       "L 10.6875 0 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-46\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-53\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_2\">\r\n",
       "     <g id=\"line2d_13\">\r\n",
       "      <path clip-path=\"url(#p3f4b1bc450)\" d=\"M 50.14375 74.098994 \r\n",
       "L 245.44375 74.098994 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_14\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"50.14375\" xlink:href=\"#mdacda6f47a\" y=\"74.098994\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_8\">\r\n",
       "      <!-- 0.10 -->\r\n",
       "      <g transform=\"translate(20.878125 77.898213)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-49\"/>\r\n",
       "       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_3\">\r\n",
       "     <g id=\"line2d_15\">\r\n",
       "      <path clip-path=\"url(#p3f4b1bc450)\" d=\"M 50.14375 35.784182 \r\n",
       "L 245.44375 35.784182 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_16\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"50.14375\" xlink:href=\"#mdacda6f47a\" y=\"35.784182\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_9\">\r\n",
       "      <!-- 0.15 -->\r\n",
       "      <g transform=\"translate(20.878125 39.583401)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-49\"/>\r\n",
       "       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-53\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"text_10\">\r\n",
       "     <!-- loss -->\r\n",
       "     <g transform=\"translate(14.798437 84.807812)rotate(-90)scale(0.1 -0.1)\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 9.421875 75.984375 \r\n",
       "L 18.40625 75.984375 \r\n",
       "L 18.40625 0 \r\n",
       "L 9.421875 0 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-108\"/>\r\n",
       "       <path d=\"M 44.28125 53.078125 \r\n",
       "L 44.28125 44.578125 \r\n",
       "Q 40.484375 46.53125 36.375 47.5 \r\n",
       "Q 32.28125 48.484375 27.875 48.484375 \r\n",
       "Q 21.1875 48.484375 17.84375 46.4375 \r\n",
       "Q 14.5 44.390625 14.5 40.28125 \r\n",
       "Q 14.5 37.15625 16.890625 35.375 \r\n",
       "Q 19.28125 33.59375 26.515625 31.984375 \r\n",
       "L 29.59375 31.296875 \r\n",
       "Q 39.15625 29.25 43.1875 25.515625 \r\n",
       "Q 47.21875 21.78125 47.21875 15.09375 \r\n",
       "Q 47.21875 7.46875 41.1875 3.015625 \r\n",
       "Q 35.15625 -1.421875 24.609375 -1.421875 \r\n",
       "Q 20.21875 -1.421875 15.453125 -0.5625 \r\n",
       "Q 10.6875 0.296875 5.421875 2 \r\n",
       "L 5.421875 11.28125 \r\n",
       "Q 10.40625 8.6875 15.234375 7.390625 \r\n",
       "Q 20.0625 6.109375 24.8125 6.109375 \r\n",
       "Q 31.15625 6.109375 34.5625 8.28125 \r\n",
       "Q 37.984375 10.453125 37.984375 14.40625 \r\n",
       "Q 37.984375 18.0625 35.515625 20.015625 \r\n",
       "Q 33.0625 21.96875 24.703125 23.78125 \r\n",
       "L 21.578125 24.515625 \r\n",
       "Q 13.234375 26.265625 9.515625 29.90625 \r\n",
       "Q 5.8125 33.546875 5.8125 39.890625 \r\n",
       "Q 5.8125 47.609375 11.28125 51.796875 \r\n",
       "Q 16.75 56 26.8125 56 \r\n",
       "Q 31.78125 56 36.171875 55.265625 \r\n",
       "Q 40.578125 54.546875 44.28125 53.078125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-115\"/>\r\n",
       "      </defs>\r\n",
       "      <use xlink:href=\"#DejaVuSans-108\"/>\r\n",
       "      <use x=\"27.783203\" xlink:href=\"#DejaVuSans-111\"/>\r\n",
       "      <use x=\"88.964844\" xlink:href=\"#DejaVuSans-115\"/>\r\n",
       "      <use x=\"141.064453\" xlink:href=\"#DejaVuSans-115\"/>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "   </g>\r\n",
       "   <g id=\"line2d_17\">\r\n",
       "    <path clip-path=\"url(#p3f4b1bc450)\" d=\"M 50.14375 13.377273 \r\n",
       "L 58.28125 55.843503 \r\n",
       "L 66.41875 79.981457 \r\n",
       "L 74.55625 95.555539 \r\n",
       "L 82.69375 106.660533 \r\n",
       "L 90.83125 115.538478 \r\n",
       "L 98.96875 120.807795 \r\n",
       "L 107.10625 124.303683 \r\n",
       "L 115.24375 127.650265 \r\n",
       "L 123.38125 129.727895 \r\n",
       "L 131.51875 131.219379 \r\n",
       "L 139.65625 132.325958 \r\n",
       "L 147.79375 133.187783 \r\n",
       "L 155.93125 134.323306 \r\n",
       "L 164.06875 134.53453 \r\n",
       "L 172.20625 135.303698 \r\n",
       "L 180.34375 135.757324 \r\n",
       "L 188.48125 135.108878 \r\n",
       "L 196.61875 136.282714 \r\n",
       "L 204.75625 136.39165 \r\n",
       "L 212.89375 136.700276 \r\n",
       "L 221.03125 136.68377 \r\n",
       "L 229.16875 136.767383 \r\n",
       "L 237.30625 136.922727 \r\n",
       "L 245.44375 136.877454 \r\n",
       "\" style=\"fill:none;stroke:#1f77b4;stroke-linecap:square;stroke-width:1.5;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_3\">\r\n",
       "    <path d=\"M 50.14375 143.1 \r\n",
       "L 50.14375 7.2 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_4\">\r\n",
       "    <path d=\"M 245.44375 143.1 \r\n",
       "L 245.44375 7.2 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_5\">\r\n",
       "    <path d=\"M 50.14375 143.1 \r\n",
       "L 245.44375 143.1 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_6\">\r\n",
       "    <path d=\"M 50.14375 7.2 \r\n",
       "L 245.44375 7.2 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "  </g>\r\n",
       " </g>\r\n",
       " <defs>\r\n",
       "  <clipPath id=\"p3f4b1bc450\">\r\n",
       "   <rect height=\"135.9\" width=\"195.3\" x=\"50.14375\" y=\"7.2\"/>\r\n",
       "  </clipPath>\r\n",
       " </defs>\r\n",
       "</svg>\r\n"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 训练\n",
    "# 设置嵌入大小、隐藏层大小、层数和丢弃率\n",
    "embed_size, num_hiddens, num_layers, dropout = 32, 32, 2, 0.1\n",
    "# 设置批量大小和序列长度\n",
    "batch_size, num_steps = 64, 10\n",
    "# 设置学习率、训练轮数和设备（GPU或CPU）\n",
    "lr, num_epochs, device = 0.005, 250, d2l.try_gpu()\n",
    "\n",
    "# 载入翻译数据集，并获取数据迭代器和词汇表\n",
    "train_iter, src_vocab, tgt_vocab = load_data_nmt(batch_size, num_steps)  \n",
    "# 创建源语言的编码器实例\n",
    "encoder = d2l.Seq2SeqEncoder(len(src_vocab), embed_size, num_hiddens, num_layers, dropout)  \n",
    "# 创建目标语言的带有注意力机制的解码器实例\n",
    "decoder = Seq2SeqAttentionDecoder(len(tgt_vocab), embed_size, num_hiddens, num_layers, dropout)   \n",
    "# 创建编码-解码模型实例\n",
    "net = d2l.EncoderDecoder(encoder, decoder)\n",
    "# 训练序列到序列模型\n",
    "d2l.train_seq2seq(net, train_iter, lr, num_epochs, tgt_vocab, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " go => va  !,  bleu 0.000\n",
      " i lost . => j'ai perdu .,  bleu 0.492\n",
      " he's calm . => il est malade .,  bleu 0.658\n",
      " i'm home . => je suis chez moi .,  bleu 1.000\n"
     ]
    }
   ],
   "source": [
    "# 将几个英语句子翻译成汉语\n",
    "# 英语句子列表\n",
    "engs = ['go', \"i lost .\", 'he\\'s calm .', 'i\\'m home .'] \n",
    "# 对应的汉语句子列表\n",
    "fras = ['va !', 'j\\' ai perdu .', 'il est calme .', 'je suis chez moi .']\n",
    "# 遍历英语句子和汉语句子的对应关系\n",
    "for eng, fra in zip(engs, fras):\n",
    "    # 使用训练好的模型net对英语句子进行翻译，并获取注意力权重序列\n",
    "    translation, dec_attention_weight_seq = d2l.predict_seq2seq(net, eng, src_vocab,\n",
    "                                                               tgt_vocab, num_steps, device, True)   \n",
    "    # 打印英语句子、翻译结果和BLEU分数\n",
    "    print(f' {eng} => {translation}, ',\n",
    "         f'bleu {d2l.bleu(translation, fra, k=2):.3f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将注意力权重序列进行拼接，并调整形状\n",
    "attention_weights = torch.cat([step[0][0][0] for step in dec_attention_weight_seq], 0).reshape((1, 1, -1, num_steps))     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\r\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n",
       "  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n",
       "<!-- Created with matplotlib (https://matplotlib.org/) -->\r\n",
       "<svg height=\"180.65625pt\" version=\"1.1\" viewBox=\"0 0 165.99575 180.65625\" width=\"165.99575pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n",
       " <metadata>\r\n",
       "  <rdf:RDF xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\r\n",
       "   <cc:Work>\r\n",
       "    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\r\n",
       "    <dc:date>2022-08-03T16:04:49.017404</dc:date>\r\n",
       "    <dc:format>image/svg+xml</dc:format>\r\n",
       "    <dc:creator>\r\n",
       "     <cc:Agent>\r\n",
       "      <dc:title>Matplotlib v3.3.4, https://matplotlib.org/</dc:title>\r\n",
       "     </cc:Agent>\r\n",
       "    </dc:creator>\r\n",
       "   </cc:Work>\r\n",
       "  </rdf:RDF>\r\n",
       " </metadata>\r\n",
       " <defs>\r\n",
       "  <style type=\"text/css\">*{stroke-linecap:butt;stroke-linejoin:round;}</style>\r\n",
       " </defs>\r\n",
       " <g id=\"figure_1\">\r\n",
       "  <g id=\"patch_1\">\r\n",
       "   <path d=\"M 0 180.65625 \r\n",
       "L 165.99575 180.65625 \r\n",
       "L 165.99575 0 \r\n",
       "L 0 0 \r\n",
       "z\r\n",
       "\" style=\"fill:none;\"/>\r\n",
       "  </g>\r\n",
       "  <g id=\"axes_1\">\r\n",
       "   <g id=\"patch_2\">\r\n",
       "    <path d=\"M 34.240625 143.1 \r\n",
       "L 124.840625 143.1 \r\n",
       "L 124.840625 7.2 \r\n",
       "L 34.240625 7.2 \r\n",
       "z\r\n",
       "\" style=\"fill:#ffffff;\"/>\r\n",
       "   </g>\r\n",
       "   <g clip-path=\"url(#p567e11aea5)\">\r\n",
       "    <image height=\"136\" id=\"imagedfb2ae66ca\" transform=\"scale(1 -1)translate(0 -136)\" width=\"91\" x=\"34.240625\" xlink:href=\"data:image/png;base64,\r\n",
       "iVBORw0KGgoAAAANSUhEUgAAAFsAAACICAYAAACInJcYAAAB80lEQVR4nO3doW1WYRiG4feQIggICLJFsAFhg3omYAEGQIMgIVgQWFRXAIFhg2pEBQkISkJSU0LaH8oA6O9W1zXAI+687jvJ2XZHr65mkeOnb5fs3ju4tWR3Zma3W5Zjri1b5j9ih8QOiR0SOyR2SOyQ2CGxQ2KHxA6JHRI7JHZI7JDYIbFDYofEDokdEjskdmg7O3yw7O3+5uHDJbvnn46X7M7MnHz+uWzbZYfEDokdEjskdkjskNghsUNih8QOiR0SOyR2SOyQ2CGxQ2KHxA6JHRI7JHZoe3799rLX9WcvH68ZPj1dszszF1++L9t22SGxQ2KHxA6JHRI7JHZI7JDYIbFDYofEDokdEjskdkjskNghsUNih8QOiR0SO7RdnZ8t+5Rh9+LJkt3Xbz4u2Z2ZOfl9uWzbZYfEDokdEjskdkjskNghsUNih8QOiR0SOyR2SOyQ2CGxQ2KHxA6JHRI7JHZI7NDenw/vlo1/e7/m3wSPDu4s2Z2Z2bZl0y67JHZI7JDYIbFDYofEDokdEjskdkjskNghsUNih8QOiR0SOyR2SOyQ2CGxQ3uzf3/Z+Ncfv5bs7t+9sWR3Zubi8u+ybZcdEjskdkjskNghsUNih8QOiR0SOyR2SOyQ2CGxQ2KHxA6JHRI7JHZI7JDYIbFD/wCeAS6azZj1OwAAAABJRU5ErkJggg==\" y=\"-7.1\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"matplotlib.axis_1\">\r\n",
       "    <g id=\"xtick_1\">\r\n",
       "     <g id=\"line2d_1\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 0 0 \r\n",
       "L 0 3.5 \r\n",
       "\" id=\"mec7cda09d4\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n",
       "      </defs>\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"45.565625\" xlink:href=\"#mec7cda09d4\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_1\">\r\n",
       "      <!-- 0 -->\r\n",
       "      <g transform=\"translate(42.384375 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 31.78125 66.40625 \r\n",
       "Q 24.171875 66.40625 20.328125 58.90625 \r\n",
       "Q 16.5 51.421875 16.5 36.375 \r\n",
       "Q 16.5 21.390625 20.328125 13.890625 \r\n",
       "Q 24.171875 6.390625 31.78125 6.390625 \r\n",
       "Q 39.453125 6.390625 43.28125 13.890625 \r\n",
       "Q 47.125 21.390625 47.125 36.375 \r\n",
       "Q 47.125 51.421875 43.28125 58.90625 \r\n",
       "Q 39.453125 66.40625 31.78125 66.40625 \r\n",
       "z\r\n",
       "M 31.78125 74.21875 \r\n",
       "Q 44.046875 74.21875 50.515625 64.515625 \r\n",
       "Q 56.984375 54.828125 56.984375 36.375 \r\n",
       "Q 56.984375 17.96875 50.515625 8.265625 \r\n",
       "Q 44.046875 -1.421875 31.78125 -1.421875 \r\n",
       "Q 19.53125 -1.421875 13.0625 8.265625 \r\n",
       "Q 6.59375 17.96875 6.59375 36.375 \r\n",
       "Q 6.59375 54.828125 13.0625 64.515625 \r\n",
       "Q 19.53125 74.21875 31.78125 74.21875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-48\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_2\">\r\n",
       "     <g id=\"line2d_2\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"90.865625\" xlink:href=\"#mec7cda09d4\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_2\">\r\n",
       "      <!-- 2 -->\r\n",
       "      <g transform=\"translate(87.684375 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 19.1875 8.296875 \r\n",
       "L 53.609375 8.296875 \r\n",
       "L 53.609375 0 \r\n",
       "L 7.328125 0 \r\n",
       "L 7.328125 8.296875 \r\n",
       "Q 12.9375 14.109375 22.625 23.890625 \r\n",
       "Q 32.328125 33.6875 34.8125 36.53125 \r\n",
       "Q 39.546875 41.84375 41.421875 45.53125 \r\n",
       "Q 43.3125 49.21875 43.3125 52.78125 \r\n",
       "Q 43.3125 58.59375 39.234375 62.25 \r\n",
       "Q 35.15625 65.921875 28.609375 65.921875 \r\n",
       "Q 23.96875 65.921875 18.8125 64.3125 \r\n",
       "Q 13.671875 62.703125 7.8125 59.421875 \r\n",
       "L 7.8125 69.390625 \r\n",
       "Q 13.765625 71.78125 18.9375 73 \r\n",
       "Q 24.125 74.21875 28.421875 74.21875 \r\n",
       "Q 39.75 74.21875 46.484375 68.546875 \r\n",
       "Q 53.21875 62.890625 53.21875 53.421875 \r\n",
       "Q 53.21875 48.921875 51.53125 44.890625 \r\n",
       "Q 49.859375 40.875 45.40625 35.40625 \r\n",
       "Q 44.1875 33.984375 37.640625 27.21875 \r\n",
       "Q 31.109375 20.453125 19.1875 8.296875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-50\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-50\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"text_3\">\r\n",
       "     <!-- key positions -->\r\n",
       "     <g transform=\"translate(46.792188 171.376563)scale(0.1 -0.1)\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 9.078125 75.984375 \r\n",
       "L 18.109375 75.984375 \r\n",
       "L 18.109375 31.109375 \r\n",
       "L 44.921875 54.6875 \r\n",
       "L 56.390625 54.6875 \r\n",
       "L 27.390625 29.109375 \r\n",
       "L 57.625 0 \r\n",
       "L 45.90625 0 \r\n",
       "L 18.109375 26.703125 \r\n",
       "L 18.109375 0 \r\n",
       "L 9.078125 0 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-107\"/>\r\n",
       "       <path d=\"M 56.203125 29.59375 \r\n",
       "L 56.203125 25.203125 \r\n",
       "L 14.890625 25.203125 \r\n",
       "Q 15.484375 15.921875 20.484375 11.0625 \r\n",
       "Q 25.484375 6.203125 34.421875 6.203125 \r\n",
       "Q 39.59375 6.203125 44.453125 7.46875 \r\n",
       "Q 49.3125 8.734375 54.109375 11.28125 \r\n",
       "L 54.109375 2.78125 \r\n",
       "Q 49.265625 0.734375 44.1875 -0.34375 \r\n",
       "Q 39.109375 -1.421875 33.890625 -1.421875 \r\n",
       "Q 20.796875 -1.421875 13.15625 6.1875 \r\n",
       "Q 5.515625 13.8125 5.515625 26.8125 \r\n",
       "Q 5.515625 40.234375 12.765625 48.109375 \r\n",
       "Q 20.015625 56 32.328125 56 \r\n",
       "Q 43.359375 56 49.78125 48.890625 \r\n",
       "Q 56.203125 41.796875 56.203125 29.59375 \r\n",
       "z\r\n",
       "M 47.21875 32.234375 \r\n",
       "Q 47.125 39.59375 43.09375 43.984375 \r\n",
       "Q 39.0625 48.390625 32.421875 48.390625 \r\n",
       "Q 24.90625 48.390625 20.390625 44.140625 \r\n",
       "Q 15.875 39.890625 15.1875 32.171875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-101\"/>\r\n",
       "       <path d=\"M 32.171875 -5.078125 \r\n",
       "Q 28.375 -14.84375 24.75 -17.8125 \r\n",
       "Q 21.140625 -20.796875 15.09375 -20.796875 \r\n",
       "L 7.90625 -20.796875 \r\n",
       "L 7.90625 -13.28125 \r\n",
       "L 13.1875 -13.28125 \r\n",
       "Q 16.890625 -13.28125 18.9375 -11.515625 \r\n",
       "Q 21 -9.765625 23.484375 -3.21875 \r\n",
       "L 25.09375 0.875 \r\n",
       "L 2.984375 54.6875 \r\n",
       "L 12.5 54.6875 \r\n",
       "L 29.59375 11.921875 \r\n",
       "L 46.6875 54.6875 \r\n",
       "L 56.203125 54.6875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-121\"/>\r\n",
       "       <path id=\"DejaVuSans-32\"/>\r\n",
       "       <path d=\"M 18.109375 8.203125 \r\n",
       "L 18.109375 -20.796875 \r\n",
       "L 9.078125 -20.796875 \r\n",
       "L 9.078125 54.6875 \r\n",
       "L 18.109375 54.6875 \r\n",
       "L 18.109375 46.390625 \r\n",
       "Q 20.953125 51.265625 25.265625 53.625 \r\n",
       "Q 29.59375 56 35.59375 56 \r\n",
       "Q 45.5625 56 51.78125 48.09375 \r\n",
       "Q 58.015625 40.1875 58.015625 27.296875 \r\n",
       "Q 58.015625 14.40625 51.78125 6.484375 \r\n",
       "Q 45.5625 -1.421875 35.59375 -1.421875 \r\n",
       "Q 29.59375 -1.421875 25.265625 0.953125 \r\n",
       "Q 20.953125 3.328125 18.109375 8.203125 \r\n",
       "z\r\n",
       "M 48.6875 27.296875 \r\n",
       "Q 48.6875 37.203125 44.609375 42.84375 \r\n",
       "Q 40.53125 48.484375 33.40625 48.484375 \r\n",
       "Q 26.265625 48.484375 22.1875 42.84375 \r\n",
       "Q 18.109375 37.203125 18.109375 27.296875 \r\n",
       "Q 18.109375 17.390625 22.1875 11.75 \r\n",
       "Q 26.265625 6.109375 33.40625 6.109375 \r\n",
       "Q 40.53125 6.109375 44.609375 11.75 \r\n",
       "Q 48.6875 17.390625 48.6875 27.296875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-112\"/>\r\n",
       "       <path d=\"M 30.609375 48.390625 \r\n",
       "Q 23.390625 48.390625 19.1875 42.75 \r\n",
       "Q 14.984375 37.109375 14.984375 27.296875 \r\n",
       "Q 14.984375 17.484375 19.15625 11.84375 \r\n",
       "Q 23.34375 6.203125 30.609375 6.203125 \r\n",
       "Q 37.796875 6.203125 41.984375 11.859375 \r\n",
       "Q 46.1875 17.53125 46.1875 27.296875 \r\n",
       "Q 46.1875 37.015625 41.984375 42.703125 \r\n",
       "Q 37.796875 48.390625 30.609375 48.390625 \r\n",
       "z\r\n",
       "M 30.609375 56 \r\n",
       "Q 42.328125 56 49.015625 48.375 \r\n",
       "Q 55.71875 40.765625 55.71875 27.296875 \r\n",
       "Q 55.71875 13.875 49.015625 6.21875 \r\n",
       "Q 42.328125 -1.421875 30.609375 -1.421875 \r\n",
       "Q 18.84375 -1.421875 12.171875 6.21875 \r\n",
       "Q 5.515625 13.875 5.515625 27.296875 \r\n",
       "Q 5.515625 40.765625 12.171875 48.375 \r\n",
       "Q 18.84375 56 30.609375 56 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-111\"/>\r\n",
       "       <path d=\"M 44.28125 53.078125 \r\n",
       "L 44.28125 44.578125 \r\n",
       "Q 40.484375 46.53125 36.375 47.5 \r\n",
       "Q 32.28125 48.484375 27.875 48.484375 \r\n",
       "Q 21.1875 48.484375 17.84375 46.4375 \r\n",
       "Q 14.5 44.390625 14.5 40.28125 \r\n",
       "Q 14.5 37.15625 16.890625 35.375 \r\n",
       "Q 19.28125 33.59375 26.515625 31.984375 \r\n",
       "L 29.59375 31.296875 \r\n",
       "Q 39.15625 29.25 43.1875 25.515625 \r\n",
       "Q 47.21875 21.78125 47.21875 15.09375 \r\n",
       "Q 47.21875 7.46875 41.1875 3.015625 \r\n",
       "Q 35.15625 -1.421875 24.609375 -1.421875 \r\n",
       "Q 20.21875 -1.421875 15.453125 -0.5625 \r\n",
       "Q 10.6875 0.296875 5.421875 2 \r\n",
       "L 5.421875 11.28125 \r\n",
       "Q 10.40625 8.6875 15.234375 7.390625 \r\n",
       "Q 20.0625 6.109375 24.8125 6.109375 \r\n",
       "Q 31.15625 6.109375 34.5625 8.28125 \r\n",
       "Q 37.984375 10.453125 37.984375 14.40625 \r\n",
       "Q 37.984375 18.0625 35.515625 20.015625 \r\n",
       "Q 33.0625 21.96875 24.703125 23.78125 \r\n",
       "L 21.578125 24.515625 \r\n",
       "Q 13.234375 26.265625 9.515625 29.90625 \r\n",
       "Q 5.8125 33.546875 5.8125 39.890625 \r\n",
       "Q 5.8125 47.609375 11.28125 51.796875 \r\n",
       "Q 16.75 56 26.8125 56 \r\n",
       "Q 31.78125 56 36.171875 55.265625 \r\n",
       "Q 40.578125 54.546875 44.28125 53.078125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-115\"/>\r\n",
       "       <path d=\"M 9.421875 54.6875 \r\n",
       "L 18.40625 54.6875 \r\n",
       "L 18.40625 0 \r\n",
       "L 9.421875 0 \r\n",
       "z\r\n",
       "M 9.421875 75.984375 \r\n",
       "L 18.40625 75.984375 \r\n",
       "L 18.40625 64.59375 \r\n",
       "L 9.421875 64.59375 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-105\"/>\r\n",
       "       <path d=\"M 18.3125 70.21875 \r\n",
       "L 18.3125 54.6875 \r\n",
       "L 36.8125 54.6875 \r\n",
       "L 36.8125 47.703125 \r\n",
       "L 18.3125 47.703125 \r\n",
       "L 18.3125 18.015625 \r\n",
       "Q 18.3125 11.328125 20.140625 9.421875 \r\n",
       "Q 21.96875 7.515625 27.59375 7.515625 \r\n",
       "L 36.8125 7.515625 \r\n",
       "L 36.8125 0 \r\n",
       "L 27.59375 0 \r\n",
       "Q 17.1875 0 13.234375 3.875 \r\n",
       "Q 9.28125 7.765625 9.28125 18.015625 \r\n",
       "L 9.28125 47.703125 \r\n",
       "L 2.6875 47.703125 \r\n",
       "L 2.6875 54.6875 \r\n",
       "L 9.28125 54.6875 \r\n",
       "L 9.28125 70.21875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-116\"/>\r\n",
       "       <path d=\"M 54.890625 33.015625 \r\n",
       "L 54.890625 0 \r\n",
       "L 45.90625 0 \r\n",
       "L 45.90625 32.71875 \r\n",
       "Q 45.90625 40.484375 42.875 44.328125 \r\n",
       "Q 39.84375 48.1875 33.796875 48.1875 \r\n",
       "Q 26.515625 48.1875 22.3125 43.546875 \r\n",
       "Q 18.109375 38.921875 18.109375 30.90625 \r\n",
       "L 18.109375 0 \r\n",
       "L 9.078125 0 \r\n",
       "L 9.078125 54.6875 \r\n",
       "L 18.109375 54.6875 \r\n",
       "L 18.109375 46.1875 \r\n",
       "Q 21.34375 51.125 25.703125 53.5625 \r\n",
       "Q 30.078125 56 35.796875 56 \r\n",
       "Q 45.21875 56 50.046875 50.171875 \r\n",
       "Q 54.890625 44.34375 54.890625 33.015625 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-110\"/>\r\n",
       "      </defs>\r\n",
       "      <use xlink:href=\"#DejaVuSans-107\"/>\r\n",
       "      <use x=\"54.285156\" xlink:href=\"#DejaVuSans-101\"/>\r\n",
       "      <use x=\"115.808594\" xlink:href=\"#DejaVuSans-121\"/>\r\n",
       "      <use x=\"174.988281\" xlink:href=\"#DejaVuSans-32\"/>\r\n",
       "      <use x=\"206.775391\" xlink:href=\"#DejaVuSans-112\"/>\r\n",
       "      <use x=\"270.251953\" xlink:href=\"#DejaVuSans-111\"/>\r\n",
       "      <use x=\"331.433594\" xlink:href=\"#DejaVuSans-115\"/>\r\n",
       "      <use x=\"383.533203\" xlink:href=\"#DejaVuSans-105\"/>\r\n",
       "      <use x=\"411.316406\" xlink:href=\"#DejaVuSans-116\"/>\r\n",
       "      <use x=\"450.525391\" xlink:href=\"#DejaVuSans-105\"/>\r\n",
       "      <use x=\"478.308594\" xlink:href=\"#DejaVuSans-111\"/>\r\n",
       "      <use x=\"539.490234\" xlink:href=\"#DejaVuSans-110\"/>\r\n",
       "      <use x=\"602.869141\" xlink:href=\"#DejaVuSans-115\"/>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "   </g>\r\n",
       "   <g id=\"matplotlib.axis_2\">\r\n",
       "    <g id=\"ytick_1\">\r\n",
       "     <g id=\"line2d_3\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 0 0 \r\n",
       "L -3.5 0 \r\n",
       "\" id=\"mc363f9e0b0\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n",
       "      </defs>\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"34.240625\" xlink:href=\"#mc363f9e0b0\" y=\"18.525\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_4\">\r\n",
       "      <!-- 0 -->\r\n",
       "      <g transform=\"translate(20.878125 22.324219)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_2\">\r\n",
       "     <g id=\"line2d_4\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"34.240625\" xlink:href=\"#mc363f9e0b0\" y=\"41.175\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_5\">\r\n",
       "      <!-- 1 -->\r\n",
       "      <g transform=\"translate(20.878125 44.974219)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 12.40625 8.296875 \r\n",
       "L 28.515625 8.296875 \r\n",
       "L 28.515625 63.921875 \r\n",
       "L 10.984375 60.40625 \r\n",
       "L 10.984375 69.390625 \r\n",
       "L 28.421875 72.90625 \r\n",
       "L 38.28125 72.90625 \r\n",
       "L 38.28125 8.296875 \r\n",
       "L 54.390625 8.296875 \r\n",
       "L 54.390625 0 \r\n",
       "L 12.40625 0 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-49\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-49\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_3\">\r\n",
       "     <g id=\"line2d_5\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"34.240625\" xlink:href=\"#mc363f9e0b0\" y=\"63.825\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_6\">\r\n",
       "      <!-- 2 -->\r\n",
       "      <g transform=\"translate(20.878125 67.624219)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-50\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_4\">\r\n",
       "     <g id=\"line2d_6\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"34.240625\" xlink:href=\"#mc363f9e0b0\" y=\"86.475\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_7\">\r\n",
       "      <!-- 3 -->\r\n",
       "      <g transform=\"translate(20.878125 90.274219)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 40.578125 39.3125 \r\n",
       "Q 47.65625 37.796875 51.625 33 \r\n",
       "Q 55.609375 28.21875 55.609375 21.1875 \r\n",
       "Q 55.609375 10.40625 48.1875 4.484375 \r\n",
       "Q 40.765625 -1.421875 27.09375 -1.421875 \r\n",
       "Q 22.515625 -1.421875 17.65625 -0.515625 \r\n",
       "Q 12.796875 0.390625 7.625 2.203125 \r\n",
       "L 7.625 11.71875 \r\n",
       "Q 11.71875 9.328125 16.59375 8.109375 \r\n",
       "Q 21.484375 6.890625 26.8125 6.890625 \r\n",
       "Q 36.078125 6.890625 40.9375 10.546875 \r\n",
       "Q 45.796875 14.203125 45.796875 21.1875 \r\n",
       "Q 45.796875 27.640625 41.28125 31.265625 \r\n",
       "Q 36.765625 34.90625 28.71875 34.90625 \r\n",
       "L 20.21875 34.90625 \r\n",
       "L 20.21875 43.015625 \r\n",
       "L 29.109375 43.015625 \r\n",
       "Q 36.375 43.015625 40.234375 45.921875 \r\n",
       "Q 44.09375 48.828125 44.09375 54.296875 \r\n",
       "Q 44.09375 59.90625 40.109375 62.90625 \r\n",
       "Q 36.140625 65.921875 28.71875 65.921875 \r\n",
       "Q 24.65625 65.921875 20.015625 65.03125 \r\n",
       "Q 15.375 64.15625 9.8125 62.3125 \r\n",
       "L 9.8125 71.09375 \r\n",
       "Q 15.4375 72.65625 20.34375 73.4375 \r\n",
       "Q 25.25 74.21875 29.59375 74.21875 \r\n",
       "Q 40.828125 74.21875 47.359375 69.109375 \r\n",
       "Q 53.90625 64.015625 53.90625 55.328125 \r\n",
       "Q 53.90625 49.265625 50.4375 45.09375 \r\n",
       "Q 46.96875 40.921875 40.578125 39.3125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-51\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-51\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_5\">\r\n",
       "     <g id=\"line2d_7\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"34.240625\" xlink:href=\"#mc363f9e0b0\" y=\"109.125\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_8\">\r\n",
       "      <!-- 4 -->\r\n",
       "      <g transform=\"translate(20.878125 112.924219)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 37.796875 64.3125 \r\n",
       "L 12.890625 25.390625 \r\n",
       "L 37.796875 25.390625 \r\n",
       "z\r\n",
       "M 35.203125 72.90625 \r\n",
       "L 47.609375 72.90625 \r\n",
       "L 47.609375 25.390625 \r\n",
       "L 58.015625 25.390625 \r\n",
       "L 58.015625 17.1875 \r\n",
       "L 47.609375 17.1875 \r\n",
       "L 47.609375 0 \r\n",
       "L 37.796875 0 \r\n",
       "L 37.796875 17.1875 \r\n",
       "L 4.890625 17.1875 \r\n",
       "L 4.890625 26.703125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-52\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-52\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_6\">\r\n",
       "     <g id=\"line2d_8\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"34.240625\" xlink:href=\"#mc363f9e0b0\" y=\"131.775\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_9\">\r\n",
       "      <!-- 5 -->\r\n",
       "      <g transform=\"translate(20.878125 135.574219)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 10.796875 72.90625 \r\n",
       "L 49.515625 72.90625 \r\n",
       "L 49.515625 64.59375 \r\n",
       "L 19.828125 64.59375 \r\n",
       "L 19.828125 46.734375 \r\n",
       "Q 21.96875 47.46875 24.109375 47.828125 \r\n",
       "Q 26.265625 48.1875 28.421875 48.1875 \r\n",
       "Q 40.625 48.1875 47.75 41.5 \r\n",
       "Q 54.890625 34.8125 54.890625 23.390625 \r\n",
       "Q 54.890625 11.625 47.5625 5.09375 \r\n",
       "Q 40.234375 -1.421875 26.90625 -1.421875 \r\n",
       "Q 22.3125 -1.421875 17.546875 -0.640625 \r\n",
       "Q 12.796875 0.140625 7.71875 1.703125 \r\n",
       "L 7.71875 11.625 \r\n",
       "Q 12.109375 9.234375 16.796875 8.0625 \r\n",
       "Q 21.484375 6.890625 26.703125 6.890625 \r\n",
       "Q 35.15625 6.890625 40.078125 11.328125 \r\n",
       "Q 45.015625 15.765625 45.015625 23.390625 \r\n",
       "Q 45.015625 31 40.078125 35.4375 \r\n",
       "Q 35.15625 39.890625 26.703125 39.890625 \r\n",
       "Q 22.75 39.890625 18.8125 39.015625 \r\n",
       "Q 14.890625 38.140625 10.796875 36.28125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-53\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-53\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"text_10\">\r\n",
       "     <!-- Query posistions -->\r\n",
       "     <g transform=\"translate(14.798438 116.949219)rotate(-90)scale(0.1 -0.1)\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 39.40625 66.21875 \r\n",
       "Q 28.65625 66.21875 22.328125 58.203125 \r\n",
       "Q 16.015625 50.203125 16.015625 36.375 \r\n",
       "Q 16.015625 22.609375 22.328125 14.59375 \r\n",
       "Q 28.65625 6.59375 39.40625 6.59375 \r\n",
       "Q 50.140625 6.59375 56.421875 14.59375 \r\n",
       "Q 62.703125 22.609375 62.703125 36.375 \r\n",
       "Q 62.703125 50.203125 56.421875 58.203125 \r\n",
       "Q 50.140625 66.21875 39.40625 66.21875 \r\n",
       "z\r\n",
       "M 53.21875 1.3125 \r\n",
       "L 66.21875 -12.890625 \r\n",
       "L 54.296875 -12.890625 \r\n",
       "L 43.5 -1.21875 \r\n",
       "Q 41.890625 -1.3125 41.03125 -1.359375 \r\n",
       "Q 40.1875 -1.421875 39.40625 -1.421875 \r\n",
       "Q 24.03125 -1.421875 14.8125 8.859375 \r\n",
       "Q 5.609375 19.140625 5.609375 36.375 \r\n",
       "Q 5.609375 53.65625 14.8125 63.9375 \r\n",
       "Q 24.03125 74.21875 39.40625 74.21875 \r\n",
       "Q 54.734375 74.21875 63.90625 63.9375 \r\n",
       "Q 73.09375 53.65625 73.09375 36.375 \r\n",
       "Q 73.09375 23.6875 67.984375 14.640625 \r\n",
       "Q 62.890625 5.609375 53.21875 1.3125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-81\"/>\r\n",
       "       <path d=\"M 8.5 21.578125 \r\n",
       "L 8.5 54.6875 \r\n",
       "L 17.484375 54.6875 \r\n",
       "L 17.484375 21.921875 \r\n",
       "Q 17.484375 14.15625 20.5 10.265625 \r\n",
       "Q 23.53125 6.390625 29.59375 6.390625 \r\n",
       "Q 36.859375 6.390625 41.078125 11.03125 \r\n",
       "Q 45.3125 15.671875 45.3125 23.6875 \r\n",
       "L 45.3125 54.6875 \r\n",
       "L 54.296875 54.6875 \r\n",
       "L 54.296875 0 \r\n",
       "L 45.3125 0 \r\n",
       "L 45.3125 8.40625 \r\n",
       "Q 42.046875 3.421875 37.71875 1 \r\n",
       "Q 33.40625 -1.421875 27.6875 -1.421875 \r\n",
       "Q 18.265625 -1.421875 13.375 4.4375 \r\n",
       "Q 8.5 10.296875 8.5 21.578125 \r\n",
       "z\r\n",
       "M 31.109375 56 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-117\"/>\r\n",
       "       <path d=\"M 41.109375 46.296875 \r\n",
       "Q 39.59375 47.171875 37.8125 47.578125 \r\n",
       "Q 36.03125 48 33.890625 48 \r\n",
       "Q 26.265625 48 22.1875 43.046875 \r\n",
       "Q 18.109375 38.09375 18.109375 28.8125 \r\n",
       "L 18.109375 0 \r\n",
       "L 9.078125 0 \r\n",
       "L 9.078125 54.6875 \r\n",
       "L 18.109375 54.6875 \r\n",
       "L 18.109375 46.1875 \r\n",
       "Q 20.953125 51.171875 25.484375 53.578125 \r\n",
       "Q 30.03125 56 36.53125 56 \r\n",
       "Q 37.453125 56 38.578125 55.875 \r\n",
       "Q 39.703125 55.765625 41.0625 55.515625 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-114\"/>\r\n",
       "      </defs>\r\n",
       "      <use xlink:href=\"#DejaVuSans-81\"/>\r\n",
       "      <use x=\"78.710938\" xlink:href=\"#DejaVuSans-117\"/>\r\n",
       "      <use x=\"142.089844\" xlink:href=\"#DejaVuSans-101\"/>\r\n",
       "      <use x=\"203.613281\" xlink:href=\"#DejaVuSans-114\"/>\r\n",
       "      <use x=\"244.726562\" xlink:href=\"#DejaVuSans-121\"/>\r\n",
       "      <use x=\"303.90625\" xlink:href=\"#DejaVuSans-32\"/>\r\n",
       "      <use x=\"335.693359\" xlink:href=\"#DejaVuSans-112\"/>\r\n",
       "      <use x=\"399.169922\" xlink:href=\"#DejaVuSans-111\"/>\r\n",
       "      <use x=\"460.351562\" xlink:href=\"#DejaVuSans-115\"/>\r\n",
       "      <use x=\"512.451172\" xlink:href=\"#DejaVuSans-105\"/>\r\n",
       "      <use x=\"540.234375\" xlink:href=\"#DejaVuSans-115\"/>\r\n",
       "      <use x=\"592.333984\" xlink:href=\"#DejaVuSans-116\"/>\r\n",
       "      <use x=\"631.542969\" xlink:href=\"#DejaVuSans-105\"/>\r\n",
       "      <use x=\"659.326172\" xlink:href=\"#DejaVuSans-111\"/>\r\n",
       "      <use x=\"720.507812\" xlink:href=\"#DejaVuSans-110\"/>\r\n",
       "      <use x=\"783.886719\" xlink:href=\"#DejaVuSans-115\"/>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_3\">\r\n",
       "    <path d=\"M 34.240625 143.1 \r\n",
       "L 34.240625 7.2 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_4\">\r\n",
       "    <path d=\"M 124.840625 143.1 \r\n",
       "L 124.840625 7.2 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_5\">\r\n",
       "    <path d=\"M 34.240625 143.1 \r\n",
       "L 124.840625 143.1 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_6\">\r\n",
       "    <path d=\"M 34.240625 7.2 \r\n",
       "L 124.840625 7.2 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "  </g>\r\n",
       "  <g id=\"axes_2\">\r\n",
       "   <g id=\"patch_7\">\r\n",
       "    <path clip-path=\"url(#p112230f22a)\" d=\"M 131.815625 115.92 \r\n",
       "L 131.815625 115.601484 \r\n",
       "L 131.815625 34.698516 \r\n",
       "L 131.815625 34.38 \r\n",
       "L 135.892625 34.38 \r\n",
       "L 135.892625 34.698516 \r\n",
       "L 135.892625 115.601484 \r\n",
       "L 135.892625 115.92 \r\n",
       "z\r\n",
       "\" style=\"fill:#ffffff;stroke:#ffffff;stroke-linejoin:miter;stroke-width:0.01;\"/>\r\n",
       "   </g>\r\n",
       "   <image height=\"81\" id=\"image6a259ec9eb\" transform=\"scale(1 -1)translate(0 -81)\" width=\"4\" x=\"132\" xlink:href=\"data:image/png;base64,\r\n",
       "iVBORw0KGgoAAAANSUhEUgAAAAQAAABRCAYAAAD1sgc6AAAAnUlEQVR4nK2SSw7DMAgFidT737SrbFIDptt6XiWSKF6O3geEt9rfZT/vZRm2gjkJcgVFIAqrVpGtIpwWTFqqIAguF6NpqT7jRAsVLrVieaBFFDdqdQ65i5yhzXA5VB/KT+cEohAwEDodX/uJFgmNIEiAjAJIgN4iCqlVy2RtUaHAViAWbzPSLodKxiD4cJexbv/HokAyOJirAuCA4gviMsq1SJ1S3wAAAABJRU5ErkJggg==\" y=\"-34\"/>\r\n",
       "   <g id=\"matplotlib.axis_3\"/>\r\n",
       "   <g id=\"matplotlib.axis_4\">\r\n",
       "    <g id=\"ytick_7\">\r\n",
       "     <g id=\"line2d_9\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 0 0 \r\n",
       "L 3.5 0 \r\n",
       "\" id=\"m8a31edc8fe\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n",
       "      </defs>\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"135.892625\" xlink:href=\"#m8a31edc8fe\" y=\"77.52936\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_11\">\r\n",
       "      <!-- 0.2 -->\r\n",
       "      <g transform=\"translate(142.892625 81.328579)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 10.6875 12.40625 \r\n",
       "L 21 12.40625 \r\n",
       "L 21 0 \r\n",
       "L 10.6875 0 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-46\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-50\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_8\">\r\n",
       "     <g id=\"line2d_10\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"135.892625\" xlink:href=\"#m8a31edc8fe\" y=\"36.838044\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_12\">\r\n",
       "      <!-- 0.4 -->\r\n",
       "      <g transform=\"translate(142.892625 40.637263)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-52\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_8\">\r\n",
       "    <path d=\"M 131.815625 115.92 \r\n",
       "L 131.815625 115.601484 \r\n",
       "L 131.815625 34.698516 \r\n",
       "L 131.815625 34.38 \r\n",
       "L 135.892625 34.38 \r\n",
       "L 135.892625 34.698516 \r\n",
       "L 135.892625 115.601484 \r\n",
       "L 135.892625 115.92 \r\n",
       "z\r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "  </g>\r\n",
       " </g>\r\n",
       " <defs>\r\n",
       "  <clipPath id=\"p567e11aea5\">\r\n",
       "   <rect height=\"135.9\" width=\"90.6\" x=\"34.240625\" y=\"7.2\"/>\r\n",
       "  </clipPath>\r\n",
       "  <clipPath id=\"p112230f22a\">\r\n",
       "   <rect height=\"81.54\" width=\"4.077\" x=\"131.815625\" y=\"34.38\"/>\r\n",
       "  </clipPath>\r\n",
       " </defs>\r\n",
       "</svg>\r\n"
      ],
      "text/plain": [
       "<Figure size 180x180 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 可视化注意力权重\n",
    "# 显示注意力权重的热图，仅显示与输入英语句子对应的位置\n",
    "d2l.show_heatmaps(attention_weights[:,:,:,:len(engs[-1].split()) + 1].cpu(),\n",
    "                 xlabel = 'key positions', ylabel = 'Query posistions')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.3",
   "language": "python",
   "name": "python3.6.3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "2041.23px",
    "left": "45.9792px",
    "top": "56px",
    "width": "301.875px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}