05.01.01.NeuralTranslation_with_SelfAttention.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Translation with Self-Attention\n",
    "\n",
    "지난 실습에서는 seq2seq 및 attention을 사용하여 날짜언어->날짜포맷 으로의 번역을 시도해 보았습니다.  \n",
    "이번 실습에서는 동일한 task를 self-attention 모델을 이용해 다시 처리해 보겠습니다.\n",
    "\n",
    "(참고)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "import unicodedata\n",
    "import re\n",
    "import time\n",
    "import shutil\n",
    "from collections import Counter\n",
    "\n",
    "# Start by importing all the things we'll need.\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.models import Model, Sequential\n",
    "from tensorflow.keras.layers import Layer, Dot, Concatenate, Input, Activation, LSTM, Dense, Embedding, CuDNNLSTM, Flatten, TimeDistributed, Dropout, LSTMCell, RNN, Bidirectional\n",
    "from keras.layers.recurrent import Recurrent\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "from tensorflow.python.keras.utils import tf_utils\n",
    "from tensorflow.keras import backend as K\n",
    "from keras.optimizers import *\n",
    "\n",
    "# This enables the Jupyter backend on some matplotlib installations.\n",
    "%matplotlib notebook\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(1984)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# csv 포맷의 데이터 경로를 지정합니다.\n",
    "train_dataset_filepath = 'datasets/nmt_date/nmt_date_train.csv'\n",
    "test_dataset_filepath = 'datasets/nmt_date/nmt_date_test.csv'\n",
    "df = pd.read_csv(train_dataset_filepath, header=None, names=['X', 'Y'])\n",
    "x_corpus = df.iloc[:,0] \n",
    "y_corpus = df.iloc[:,1] \n",
    "x_corpus_list = x_corpus.values.tolist()\n",
    "y_corpus_list = y_corpus.values.tolist()\n",
    "x_char_list = np.concatenate([list(tuple(x)) for x in x_corpus_list], axis=0)\n",
    "y_char_list = np.concatenate([list(tuple(y)) for y in y_corpus_list], axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter_x = Counter(x_char_list)\n",
    "counter_y = Counter(y_char_list)\n",
    "x_vocab = ['<PAD>', '<UNK>', '<S>', '</S>']\n",
    "x_vocab = x_vocab + list(Counter(dict(counter_x.most_common())))\n",
    "y_vocab = ['<PAD>', '<UNK>', '<S>', '</S>']\n",
    "y_vocab = y_vocab + list(Counter(dict(counter_y.most_common())))\n",
    "idx2char_x = dict(enumerate(x_vocab))\n",
    "char2idx_x = {char:index for index, char in enumerate(x_vocab)}\n",
    "idx2char_y = dict(enumerate(y_vocab))\n",
    "char2idx_y = {char:index for index, char in enumerate(y_vocab)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare Train Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_sentence_to_indexed_corpus(corpus, char2idx):\n",
    "    indexed_corpus = [char2idx['<S>']]\n",
    "    indexed_corpus = indexed_corpus + [char2idx[char] if char in char2idx else char2idx['UNK'] for char in tuple(corpus)]\n",
    "    indexed_corpus = indexed_corpus + [char2idx_x['</S>']]\n",
    "    return indexed_corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "indexed_x_corpus_list = []\n",
    "for doc in x_corpus_list:\n",
    "    indexed_x_corpus_list.append(convert_sentence_to_indexed_corpus(doc, char2idx_x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "indexed_y_corpus_list = []\n",
    "for doc in y_corpus_list:\n",
    "    indexed_y_corpus_list.append(convert_sentence_to_indexed_corpus(doc, char2idx_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_x_corpus_length = max([len(doc) for doc in indexed_x_corpus_list])\n",
    "max_y_corpus_length = max([len(doc) for doc in indexed_y_corpus_list])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(500000, 68)\n",
      "(500000, 12)\n",
      "(500000, 12, 1)\n"
     ]
    }
   ],
   "source": [
    "input_data = tf.keras.preprocessing.sequence.pad_sequences(indexed_x_corpus_list, maxlen=max_x_corpus_length, padding=\"post\")\n",
    "output_data = tf.keras.preprocessing.sequence.pad_sequences(indexed_y_corpus_list, maxlen=max_y_corpus_length, padding=\"post\")\n",
    "teacher_data = output_data\n",
    "\n",
    "target_data = [[teacher_data[n][i+1] for i in range(len(teacher_data[n])-1)] for n in range(len(teacher_data))]\n",
    "target_data = tf.keras.preprocessing.sequence.pad_sequences(target_data, maxlen=max_y_corpus_length, padding=\"post\")\n",
    "target_data = target_data.reshape((target_data.shape[0], target_data.shape[1], 1))\n",
    "\n",
    "print(input_data.shape)\n",
    "print(teacher_data.shape)\n",
    "print(target_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "BUFFER_SIZE = len(x_corpus_list)\n",
    "BATCH_SIZE = 32\n",
    "embedding_dim = 16\n",
    "units = 32\n",
    "x_vocab_size = len(idx2char_y)\n",
    "y_vocab_size = len(idx2char_y)\n",
    "len_input = max_x_corpus_length\n",
    "len_target = max_y_corpus_length"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Transformer model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformer import Transformer, LRSchedulerPerStep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "d_model = 32   # 256?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TokenList:\n",
    "\tdef __init__(self, token_list):\n",
    "\t\tself.id2t = ['<PAD>', '<UNK>', '<S>', '</S>'] + token_list\n",
    "\t\tself.t2id = {v:k for k,v in enumerate(self.id2t)}\n",
    "\tdef id(self, x):\treturn self.t2id.get(x, 1)\n",
    "\tdef token(self, x):\treturn self.id2t[x]\n",
    "\tdef num(self):\t\treturn len(self.id2t)\n",
    "\tdef startid(self):  return 2\n",
    "\tdef endid(self):    return 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "itokens = TokenList(list(Counter(dict(counter_x.most_common()))))\n",
    "otokens = TokenList(list(Counter(dict(counter_y.most_common()))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\uhmpp\\Anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From C:\\Users\\uhmpp\\Anaconda3\\envs\\py36\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)               (None, None)         0           input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "input_1 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lambda_5 (Lambda)               (None, None)         0           lambda_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_4 (Lambda)               (None, None)         0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_3 (Embedding)         (None, None, 32)     480         lambda_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, None, 32)     2240        lambda_4[0][0]                   \n",
      "                                                                 lambda_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, None, 32)     44832       input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, None, 32)     0           embedding_2[0][0]                \n",
      "                                                                 embedding_1[0][0]                \n",
      "                                                                 embedding_3[0][0]                \n",
      "                                                                 embedding_1[1][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, None, 32)     0           lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, None, 32)     1024        dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, None, 32)     1024        dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_6 (Lambda)               (None, None)         0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_7 (Lambda)               (None, None, None)   0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_8 (Lambda)               (None, None, None)   0           dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_10 (Lambda)              (None, None)         0           lambda_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_11 (Lambda)              (None, None, None)   0           lambda_7[0][0]                   \n",
      "                                                                 lambda_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_12 (Lambda)              (None, None)         0           lambda_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, None, None)   0           lambda_11[0][0]                  \n",
      "                                                                 lambda_12[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, None, None)   0           add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, None, 32)     1024        dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, None, None)   0           activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lambda_9 (Lambda)               (None, None, None)   0           dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_13 (Lambda)              (None, None, None)   0           dropout_2[0][0]                  \n",
      "                                                                 lambda_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_14 (Lambda)              (None, None, 32)     0           lambda_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, None, 32)     1056        lambda_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_12 (Dropout)            (None, None, 32)     0           time_distributed_1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, None, 32)     0           dropout_1[0][0]                  \n",
      "                                                                 dropout_12[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_2 (LayerNor (None, None, 32)     64          add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, None, 512)    16896       layer_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, None, 32)     16416       conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, None, 32)     0           conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, None, 32)     0           dropout_3[0][0]                  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                 layer_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_1 (LayerNor (None, None, 32)     64          add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, None, 32)     1024        layer_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, None, 32)     1024        layer_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "lambda_15 (Lambda)              (None, None, None)   0           dense_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_16 (Lambda)              (None, None, None)   0           dense_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_18 (Lambda)              (None, None)         0           lambda_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_19 (Lambda)              (None, None, None)   0           lambda_15[0][0]                  \n",
      "                                                                 lambda_16[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_20 (Lambda)              (None, None)         0           lambda_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, None, None)   0           lambda_19[0][0]                  \n",
      "                                                                 lambda_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_23 (Lambda)              (None, None, None)   0           lambda_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_24 (Lambda)              (None, None, None)   0           lambda_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, None, None)   0           add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, None, 32)     1024        layer_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, None, 32)     1024        lambda_1[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_10 (Dense)                (None, None, 32)     1024        lambda_1[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_25 (Lambda)              (None, None, None)   0           lambda_23[0][0]                  \n",
      "                                                                 lambda_24[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, None, None)   0           activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lambda_17 (Lambda)              (None, None, None)   0           dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_27 (Lambda)              (None, None, None)   0           dense_9[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_28 (Lambda)              (None, None, None)   0           dense_10[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_30 (Lambda)              (None, None, None)   0           lambda_25[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_21 (Lambda)              (None, None, None)   0           dropout_4[0][0]                  \n",
      "                                                                 lambda_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_31 (Lambda)              (None, None, None)   0           lambda_27[0][0]                  \n",
      "                                                                 lambda_28[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_32 (Lambda)              (None, None, None)   0           lambda_30[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_22 (Lambda)              (None, None, 32)     0           lambda_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, None, None)   0           lambda_31[0][0]                  \n",
      "                                                                 lambda_32[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_2 (TimeDistrib (None, None, 32)     1056        lambda_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, None, None)   0           add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_11 (Dense)                (None, None, 32)     1024        lambda_1[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_13 (Dropout)            (None, None, 32)     0           time_distributed_2[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)             (None, None, None)   0           activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lambda_29 (Lambda)              (None, None, None)   0           dense_11[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, None, 32)     0           layer_normalization_1[0][0]      \n",
      "                                                                 dropout_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_33 (Lambda)              (None, None, None)   0           dropout_6[0][0]                  \n",
      "                                                                 lambda_29[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_4 (LayerNor (None, None, 32)     64          add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "lambda_34 (Lambda)              (None, None, 32)     0           lambda_33[0][0]                  \n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conv1d_3 (Conv1D)               (None, None, 512)    16896       layer_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_3 (TimeDistrib (None, None, 32)     1056        lambda_34[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_4 (Conv1D)               (None, None, 32)     16416       conv1d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_14 (Dropout)            (None, None, 32)     0           time_distributed_3[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, None, 32)     0           conv1d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, None, 32)     0           lambda_1[1][0]                   \n",
      "                                                                 dropout_14[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, None, 32)     0           dropout_5[0][0]                  \n",
      "                                                                 layer_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_6 (LayerNor (None, None, 32)     64          add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_3 (LayerNor (None, None, 32)     64          add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_13 (Dense)                (None, None, 32)     1024        layer_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_14 (Dense)                (None, None, 32)     1024        layer_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "lambda_26 (Lambda)              (None, None, None)   0           lambda_2[0][0]                   \n",
      "                                                                 input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_35 (Lambda)              (None, None, None)   0           dense_13[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_36 (Lambda)              (None, None, None)   0           dense_14[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_38 (Lambda)              (None, None, None)   0           lambda_26[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_39 (Lambda)              (None, None, None)   0           lambda_35[0][0]                  \n",
      "                                                                 lambda_36[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_40 (Lambda)              (None, None, None)   0           lambda_38[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_9 (Add)                     (None, None, None)   0           lambda_39[0][0]                  \n",
      "                                                                 lambda_40[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, None, None)   0           add_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (None, None, 32)     1024        layer_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dropout_7 (Dropout)             (None, None, None)   0           activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lambda_37 (Lambda)              (None, None, None)   0           dense_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_41 (Lambda)              (None, None, None)   0           dropout_7[0][0]                  \n",
      "                                                                 lambda_37[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_42 (Lambda)              (None, None, 32)     0           lambda_41[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_4 (TimeDistrib (None, None, 32)     1056        lambda_42[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_15 (Dropout)            (None, None, 32)     0           time_distributed_4[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "add_10 (Add)                    (None, None, 32)     0           layer_normalization_6[0][0]      \n",
      "                                                                 dropout_15[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_7 (LayerNor (None, None, 32)     64          add_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_5 (Conv1D)               (None, None, 512)    16896       layer_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_6 (Conv1D)               (None, None, 32)     16416       conv1d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_8 (Dropout)             (None, None, 32)     0           conv1d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_11 (Add)                    (None, None, 32)     0           dropout_8[0][0]                  \n",
      "                                                                 layer_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_5 (LayerNor (None, None, 32)     64          add_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_17 (Dense)                (None, None, 32)     1024        layer_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_18 (Dense)                (None, None, 32)     1024        layer_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "lambda_43 (Lambda)              (None, None, None)   0           dense_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_44 (Lambda)              (None, None, None)   0           dense_18[0][0]                   \n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lambda_46 (Lambda)              (None, None, None)   0           lambda_25[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_47 (Lambda)              (None, None, None)   0           lambda_43[0][0]                  \n",
      "                                                                 lambda_44[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_48 (Lambda)              (None, None, None)   0           lambda_46[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_12 (Add)                    (None, None, None)   0           lambda_47[0][0]                  \n",
      "                                                                 lambda_48[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, None, None)   0           add_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_19 (Dense)                (None, None, 32)     1024        layer_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dropout_9 (Dropout)             (None, None, None)   0           activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lambda_45 (Lambda)              (None, None, None)   0           dense_19[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_49 (Lambda)              (None, None, None)   0           dropout_9[0][0]                  \n",
      "                                                                 lambda_45[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_50 (Lambda)              (None, None, 32)     0           lambda_49[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_5 (TimeDistrib (None, None, 32)     1056        lambda_50[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_16 (Dropout)            (None, None, 32)     0           time_distributed_5[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "add_13 (Add)                    (None, None, 32)     0           layer_normalization_5[0][0]      \n",
      "                                                                 dropout_16[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_9 (LayerNor (None, None, 32)     64          add_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_21 (Dense)                (None, None, 32)     1024        layer_normalization_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_22 (Dense)                (None, None, 32)     1024        layer_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "lambda_51 (Lambda)              (None, None, None)   0           dense_21[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_52 (Lambda)              (None, None, None)   0           dense_22[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_54 (Lambda)              (None, None, None)   0           lambda_26[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_55 (Lambda)              (None, None, None)   0           lambda_51[0][0]                  \n",
      "                                                                 lambda_52[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_56 (Lambda)              (None, None, None)   0           lambda_54[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_14 (Add)                    (None, None, None)   0           lambda_55[0][0]                  \n",
      "                                                                 lambda_56[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, None, None)   0           add_14[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_23 (Dense)                (None, None, 32)     1024        layer_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dropout_10 (Dropout)            (None, None, None)   0           activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lambda_53 (Lambda)              (None, None, None)   0           dense_23[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_57 (Lambda)              (None, None, None)   0           dropout_10[0][0]                 \n",
      "                                                                 lambda_53[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_58 (Lambda)              (None, None, 32)     0           lambda_57[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_6 (TimeDistrib (None, None, 32)     1056        lambda_58[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_17 (Dropout)            (None, None, 32)     0           time_distributed_6[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "add_15 (Add)                    (None, None, 32)     0           layer_normalization_9[0][0]      \n",
      "                                                                 dropout_17[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_10 (LayerNo (None, None, 32)     64          add_15[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_7 (Conv1D)               (None, None, 512)    16896       layer_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_8 (Conv1D)               (None, None, 32)     16416       conv1d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_11 (Dropout)            (None, None, 32)     0           conv1d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_16 (Add)                    (None, None, 32)     0           dropout_11[0][0]                 \n",
      "                                                                 layer_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_8 (LayerNor (None, None, 32)     64          add_16[0][0]                     \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "time_distributed_7 (TimeDistrib (None, None, 15)     480         layer_normalization_8[0][0]      \n",
      "==================================================================================================\n",
      "Total params: 206,688\n",
      "Trainable params: 204,448\n",
      "Non-trainable params: 2,240\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "s2s = Transformer(itokens, otokens, len_limit=70, d_model=d_model, d_inner_hid=512, n_head=8, layers=2, dropout=0.1)\n",
    "lr_scheduler = LRSchedulerPerStep(d_model, 4000) \n",
    "# model_saver = ModelCheckpoint(mfile, save_best_only=True, save_weights_only=True)\n",
    "\n",
    "s2s.compile(Adam(0.001, 0.9, 0.98, epsilon=1e-9))\n",
    "s2s.model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\uhmpp\\Anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From C:\\Users\\uhmpp\\Anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "Train on 400000 samples, validate on 100000 samples\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  2280/400000 [..............................] - ETA: 80:41:57 - loss: 3.0538 - ppl: 21.1952 - accu: 0.113 - ETA: 27:12:28 - loss: 2.8629 - ppl: 17.7311 - accu: 0.162 - ETA: 16:29:45 - loss: 2.7420 - ppl: 15.8637 - accu: 0.152 - ETA: 11:54:46 - loss: 2.6944 - ppl: 15.0922 - accu: 0.162 - ETA: 9:22:11 - loss: 2.6545 - ppl: 14.4867 - accu: 0.164 - ETA: 7:45:59 - loss: 2.5931 - ppl: 13.6979 - accu: 0.16 - ETA: 6:38:19 - loss: 2.5513 - ppl: 13.1578 - accu: 0.17 - ETA: 5:48:57 - loss: 2.5259 - ppl: 12.8173 - accu: 0.17 - ETA: 5:11:16 - loss: 2.4980 - ppl: 12.4700 - accu: 0.17 - ETA: 4:41:30 - loss: 2.4772 - ppl: 12.2088 - accu: 0.18 - ETA: 4:17:22 - loss: 2.4503 - ppl: 11.9023 - accu: 0.18 - ETA: 3:57:17 - loss: 2.4219 - ppl: 11.5950 - accu: 0.18 - ETA: 3:40:31 - loss: 2.4023 - ppl: 11.3737 - accu: 0.19 - ETA: 3:26:09 - loss: 2.3854 - ppl: 11.1829 - accu: 0.19 - ETA: 3:13:45 - loss: 2.3656 - ppl: 10.9752 - accu: 0.19 - ETA: 3:02:56 - loss: 2.3487 - ppl: 10.7960 - accu: 0.20 - ETA: 2:53:33 - loss: 2.3352 - ppl: 10.6496 - accu: 0.20 - ETA: 2:45:02 - loss: 2.3177 - ppl: 10.4757 - accu: 0.21 - ETA: 2:37:22 - loss: 2.3024 - ppl: 10.3234 - accu: 0.21 - ETA: 2:30:27 - loss: 2.2847 - ppl: 10.1577 - accu: 0.22 - ETA: 2:24:21 - loss: 2.2692 - ppl: 10.0111 - accu: 0.22 - ETA: 2:18:54 - loss: 2.2534 - ppl: 9.8661 - accu: 0.2339 - ETA: 2:13:44 - loss: 2.2368 - ppl: 9.7193 - accu: 0.241 - ETA: 2:09:01 - loss: 2.2196 - ppl: 9.5716 - accu: 0.247 - ETA: 2:04:46 - loss: 2.2028 - ppl: 9.4303 - accu: 0.253 - ETA: 2:00:51 - loss: 2.1854 - ppl: 9.2889 - accu: 0.257 - ETA: 1:57:11 - loss: 2.1697 - ppl: 9.1598 - accu: 0.260 - ETA: 1:53:52 - loss: 2.1526 - ppl: 9.0262 - accu: 0.265 - ETA: 1:50:45 - loss: 2.1383 - ppl: 8.9106 - accu: 0.271 - ETA: 1:47:51 - loss: 2.1223 - ppl: 8.7881 - accu: 0.277 - ETA: 1:45:07 - loss: 2.1080 - ppl: 8.6767 - accu: 0.281 - ETA: 1:42:31 - loss: 2.0911 - ppl: 8.5550 - accu: 0.289 - ETA: 1:40:09 - loss: 2.0750 - ppl: 8.4397 - accu: 0.294 - ETA: 1:37:54 - loss: 2.0593 - ppl: 8.3282 - accu: 0.299 - ETA: 1:35:46 - loss: 2.0460 - ppl: 8.2305 - accu: 0.302 - ETA: 1:33:49 - loss: 2.0310 - ppl: 8.1268 - accu: 0.308 - ETA: 1:31:53 - loss: 2.0186 - ppl: 8.0371 - accu: 0.312 - ETA: 1:30:04 - loss: 2.0055 - ppl: 7.9468 - accu: 0.317 - ETA: 1:28:20 - loss: 1.9901 - ppl: 7.8470 - accu: 0.322 - ETA: 1:26:45 - loss: 1.9770 - ppl: 7.7591 - accu: 0.328 - ETA: 1:25:13 - loss: 1.9619 - ppl: 7.6642 - accu: 0.334 - ETA: 1:23:44 - loss: 1.9481 - ppl: 7.5765 - accu: 0.340 - ETA: 1:22:18 - loss: 1.9342 - ppl: 7.4896 - accu: 0.346 - ETA: 1:20:57 - loss: 1.9207 - ppl: 7.4061 - accu: 0.351 - ETA: 1:19:39 - loss: 1.9075 - ppl: 7.3249 - accu: 0.356 - ETA: 1:18:26 - loss: 1.8946 - ppl: 7.2463 - accu: 0.360 - ETA: 1:17:15 - loss: 1.8818 - ppl: 7.1696 - accu: 0.365 - ETA: 1:16:08 - loss: 1.8699 - ppl: 7.0972 - accu: 0.369 - ETA: 1:15:01 - loss: 1.8580 - ppl: 7.0262 - accu: 0.373 - ETA: 1:13:58 - loss: 1.8440 - ppl: 6.9491 - accu: 0.377 - ETA: 1:12:59 - loss: 1.8326 - ppl: 6.8817 - accu: 0.379 - ETA: 1:12:02 - loss: 1.8208 - ppl: 6.8143 - accu: 0.383 - ETA: 1:11:07 - loss: 1.8082 - ppl: 6.7452 - accu: 0.386 - ETA: 1:10:15 - loss: 1.7959 - ppl: 6.6781 - accu: 0.390 - ETA: 1:09:25 - loss: 1.7832 - ppl: 6.6111 - accu: 0.393 - ETA: 1:08:36 - loss: 1.7714 - ppl: 6.5476 - accu: 0.397 - ETA: 1:07:55 - loss: 1.7611 - ppl: 6.4900 - accu: 0.400 - ETA: 1:07:15 - loss: 1.7505 - ppl: 6.4320 - accu: 0.404 - ETA: 1:06:34 - loss: 1.7396 - ppl: 6.3741 - accu: 0.407 - ETA: 1:05:51 - loss: 1.7303 - ppl: 6.3221 - accu: 0.409 - ETA: 1:05:11 - loss: 1.7204 - ppl: 6.2688 - accu: 0.412 - ETA: 1:04:29 - loss: 1.7103 - ppl: 6.2159 - accu: 0.415 - ETA: 1:03:53 - loss: 1.7007 - ppl: 6.1649 - accu: 0.418 - ETA: 1:03:15 - loss: 1.6907 - ppl: 6.1135 - accu: 0.420 - ETA: 1:02:37 - loss: 1.6807 - ppl: 6.0627 - accu: 0.423 - ETA: 1:02:01 - loss: 1.6714 - ppl: 6.0150 - accu: 0.425 - ETA: 1:01:27 - loss: 1.6630 - ppl: 5.9703 - accu: 0.427 - ETA: 1:00:53 - loss: 1.6546 - ppl: 5.9263 - accu: 0.429 - ETA: 1:00:20 - loss: 1.6459 - ppl: 5.8819 - accu: 0.432 - ETA: 59:48 - loss: 1.6376 - ppl: 5.8390 - accu: 0.4343  - ETA: 59:17 - loss: 1.6284 - ppl: 5.7944 - accu: 0.436 - ETA: 58:49 - loss: 1.6206 - ppl: 5.7541 - accu: 0.438 - ETA: 58:19 - loss: 1.6132 - ppl: 5.7157 - accu: 0.440 - ETA: 57:51 - loss: 1.6053 - ppl: 5.6761 - accu: 0.441 - ETA: 57:25 - loss: 1.5973 - ppl: 5.6367 - accu: 0.443 - ETA: 56:59 - loss: 1.5897 - ppl: 5.5989 - accu: 0.445 - ETA: 56:36 - loss: 1.5843 - ppl: 5.5682 - accu: 0.447 - ETA: 56:10 - loss: 1.5776 - ppl: 5.5337 - accu: 0.448 - ETA: 55:47 - loss: 1.5706 - ppl: 5.4989 - accu: 0.450 - ETA: 55:24 - loss: 1.5639 - ppl: 5.4651 - accu: 0.452 - ETA: 55:04 - loss: 1.5575 - ppl: 5.4328 - accu: 0.453 - ETA: 54:41 - loss: 1.5504 - ppl: 5.3988 - accu: 0.456 - ETA: 54:19 - loss: 1.5444 - ppl: 5.3682 - accu: 0.457 - ETA: 53:59 - loss: 1.5380 - ppl: 5.3369 - accu: 0.458 - ETA: 53:38 - loss: 1.5316 - ppl: 5.3057 - accu: 0.459 - ETA: 53:17 - loss: 1.5259 - ppl: 5.2768 - accu: 0.460 - ETA: 52:56 - loss: 1.5199 - ppl: 5.2475 - accu: 0.462 - ETA: 52:36 - loss: 1.5143 - ppl: 5.2196 - accu: 0.463 - ETA: 52:19 - loss: 1.5087 - ppl: 5.1919 - accu: 0.464 - ETA: 52:00 - loss: 1.5028 - ppl: 5.1636 - accu: 0.466 - ETA: 51:43 - loss: 1.4979 - ppl: 5.1387 - accu: 0.467 - ETA: 51:25 - loss: 1.4930 - ppl: 5.1134 - accu: 0.468 - ETA: 51:10 - loss: 1.4879 - ppl: 5.0882 - accu: 0.469 - ETA: 50:55 - loss: 1.4823 - ppl: 5.0621 - accu: 0.471 - ETA: 50:40 - loss: 1.4773 - ppl: 5.0374 - accu: 0.472 - ETA: 50:25 - loss: 1.4727 - ppl: 5.0142 - accu: 0.473 - ETA: 50:09 - loss: 1.4676 - ppl: 4.9898 - accu: 0.474 - ETA: 49:55 - loss: 1.4627 - ppl: 4.9664 - accu: 0.476 - ETA: 49:39 - loss: 1.4579 - ppl: 4.9434 - accu: 0.476 - ETA: 49:23 - loss: 1.4540 - ppl: 4.9229 - accu: 0.477 - ETA: 49:09 - loss: 1.4492 - ppl: 4.9001 - accu: 0.478 - ETA: 48:55 - loss: 1.4444 - ppl: 4.8778 - accu: 0.480 - ETA: 48:39 - loss: 1.4403 - ppl: 4.8572 - accu: 0.481 - ETA: 48:25 - loss: 1.4361 - ppl: 4.8368 - accu: 0.481 - ETA: 48:11 - loss: 1.4320 - ppl: 4.8166 - accu: 0.482 - ETA: 47:56 - loss: 1.4281 - ppl: 4.7975 - accu: 0.483 - ETA: 47:44 - loss: 1.4238 - ppl: 4.7772 - accu: 0.484 - ETA: 47:31 - loss: 1.4201 - ppl: 4.7585 - accu: 0.484 - ETA: 47:19 - loss: 1.4158 - ppl: 4.7388 - accu: 0.485 - ETA: 47:09 - loss: 1.4122 - ppl: 4.7208 - accu: 0.486 - ETA: 47:00 - loss: 1.4086 - ppl: 4.7031 - accu: 0.487 - ETA: 46:49 - loss: 1.4049 - ppl: 4.6853 - accu: 0.488 - ETA: 46:38 - loss: 1.4009 - ppl: 4.6667 - accu: 0.489 - ETA: 46:26 - loss: 1.3971 - ppl: 4.6487 - accu: 0.490 - ETA: 46:14 - loss: 1.3933 - ppl: 4.6309 - accu: 0.490 - ETA: 46:03 - loss: 1.3902 - ppl: 4.6151 - accu: 0.491 - ETA: 45:52 - loss: 1.3867 - ppl: 4.5987 - accu: 0.492 - ETA: 45:40 - loss: 1.3836 - ppl: 4.5831 - accu: 0.493 - ETA: 45:29 - loss: 1.3800 - ppl: 4.5664 - accu: 0.494 - ETA: 45:18 - loss: 1.3767 - ppl: 4.5507 - accu: 0.495 - ETA: 45:08 - loss: 1.3734 - ppl: 4.5350 - accu: 0.495 - ETA: 44:56 - loss: 1.3701 - ppl: 4.5193 - accu: 0.496 - ETA: 44:46 - loss: 1.3669 - ppl: 4.5042 - accu: 0.497 - ETA: 44:37 - loss: 1.3642 - ppl: 4.4905 - accu: 0.497 - ETA: 44:29 - loss: 1.3608 - ppl: 4.4751 - accu: 0.498 - ETA: 44:21 - loss: 1.3577 - ppl: 4.4604 - accu: 0.499 - ETA: 44:15 - loss: 1.3546 - ppl: 4.4459 - accu: 0.499 - ETA: 44:08 - loss: 1.3516 - ppl: 4.4318 - accu: 0.500 - ETA: 44:01 - loss: 1.3485 - ppl: 4.4175 - accu: 0.501 - ETA: 43:53 - loss: 1.3454 - ppl: 4.4033 - accu: 0.501 - ETA: 43:45 - loss: 1.3423 - ppl: 4.3894 - accu: 0.501 - ETA: 43:38 - loss: 1.3396 - ppl: 4.3763 - accu: 0.502 - ETA: 43:30 - loss: 1.3365 - ppl: 4.3625 - accu: 0.502 - ETA: 43:22 - loss: 1.3341 - ppl: 4.3505 - accu: 0.503 - ETA: 43:14 - loss: 1.3312 - ppl: 4.3373 - accu: 0.503 - ETA: 43:06 - loss: 1.3284 - ppl: 4.3243 - accu: 0.504 - ETA: 42:57 - loss: 1.3255 - ppl: 4.3112 - accu: 0.505 - ETA: 42:49 - loss: 1.3229 - ppl: 4.2991 - accu: 0.506 - ETA: 42:42 - loss: 1.3207 - ppl: 4.2880 - accu: 0.506 - ETA: 42:33 - loss: 1.3181 - ppl: 4.2760 - accu: 0.507 - ETA: 42:25 - loss: 1.3156 - ppl: 4.2642 - accu: 0.508 - ETA: 42:17 - loss: 1.3133 - ppl: 4.2532 - accu: 0.508 - ETA: 42:11 - loss: 1.3109 - ppl: 4.2419 - accu: 0.5092"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  4616/400000 [..............................] - ETA: 42:05 - loss: 1.3083 - ppl: 4.2301 - accu: 0.509 - ETA: 41:58 - loss: 1.3062 - ppl: 4.2197 - accu: 0.509 - ETA: 41:50 - loss: 1.3037 - ppl: 4.2082 - accu: 0.510 - ETA: 41:44 - loss: 1.3013 - ppl: 4.1972 - accu: 0.511 - ETA: 41:37 - loss: 1.2990 - ppl: 4.1866 - accu: 0.511 - ETA: 41:32 - loss: 1.2966 - ppl: 4.1758 - accu: 0.512 - ETA: 41:27 - loss: 1.2943 - ppl: 4.1650 - accu: 0.513 - ETA: 41:21 - loss: 1.2921 - ppl: 4.1547 - accu: 0.513 - ETA: 41:14 - loss: 1.2902 - ppl: 4.1453 - accu: 0.514 - ETA: 41:08 - loss: 1.2879 - ppl: 4.1352 - accu: 0.514 - ETA: 41:01 - loss: 1.2859 - ppl: 4.1254 - accu: 0.514 - ETA: 40:55 - loss: 1.2837 - ppl: 4.1155 - accu: 0.515 - ETA: 40:48 - loss: 1.2815 - ppl: 4.1055 - accu: 0.515 - ETA: 40:41 - loss: 1.2796 - ppl: 4.0962 - accu: 0.516 - ETA: 40:34 - loss: 1.2777 - ppl: 4.0871 - accu: 0.516 - ETA: 40:29 - loss: 1.2755 - ppl: 4.0775 - accu: 0.517 - ETA: 40:24 - loss: 1.2736 - ppl: 4.0685 - accu: 0.517 - ETA: 40:20 - loss: 1.2716 - ppl: 4.0594 - accu: 0.518 - ETA: 40:18 - loss: 1.2700 - ppl: 4.0511 - accu: 0.518 - ETA: 40:14 - loss: 1.2680 - ppl: 4.0421 - accu: 0.518 - ETA: 40:09 - loss: 1.2661 - ppl: 4.0333 - accu: 0.519 - ETA: 40:05 - loss: 1.2642 - ppl: 4.0245 - accu: 0.519 - ETA: 40:00 - loss: 1.2623 - ppl: 4.0160 - accu: 0.520 - ETA: 39:54 - loss: 1.2604 - ppl: 4.0071 - accu: 0.520 - ETA: 39:48 - loss: 1.2587 - ppl: 3.9991 - accu: 0.521 - ETA: 39:42 - loss: 1.2572 - ppl: 3.9916 - accu: 0.521 - ETA: 39:36 - loss: 1.2555 - ppl: 3.9836 - accu: 0.522 - ETA: 39:30 - loss: 1.2536 - ppl: 3.9751 - accu: 0.522 - ETA: 39:25 - loss: 1.2517 - ppl: 3.9669 - accu: 0.523 - ETA: 39:19 - loss: 1.2500 - ppl: 3.9589 - accu: 0.523 - ETA: 39:14 - loss: 1.2483 - ppl: 3.9511 - accu: 0.524 - ETA: 39:09 - loss: 1.2462 - ppl: 3.9423 - accu: 0.524 - ETA: 39:03 - loss: 1.2447 - ppl: 3.9349 - accu: 0.525 - ETA: 38:59 - loss: 1.2433 - ppl: 3.9280 - accu: 0.525 - ETA: 38:53 - loss: 1.2418 - ppl: 3.9211 - accu: 0.525 - ETA: 38:49 - loss: 1.2402 - ppl: 3.9135 - accu: 0.525 - ETA: 38:44 - loss: 1.2389 - ppl: 3.9070 - accu: 0.526 - ETA: 38:39 - loss: 1.2375 - ppl: 3.9001 - accu: 0.526 - ETA: 38:34 - loss: 1.2360 - ppl: 3.8931 - accu: 0.526 - ETA: 38:30 - loss: 1.2346 - ppl: 3.8864 - accu: 0.526 - ETA: 38:25 - loss: 1.2332 - ppl: 3.8798 - accu: 0.527 - ETA: 38:20 - loss: 1.2317 - ppl: 3.8727 - accu: 0.527 - ETA: 38:15 - loss: 1.2303 - ppl: 3.8663 - accu: 0.527 - ETA: 38:10 - loss: 1.2290 - ppl: 3.8600 - accu: 0.527 - ETA: 38:06 - loss: 1.2274 - ppl: 3.8529 - accu: 0.527 - ETA: 38:01 - loss: 1.2260 - ppl: 3.8463 - accu: 0.528 - ETA: 37:56 - loss: 1.2247 - ppl: 3.8399 - accu: 0.528 - ETA: 37:52 - loss: 1.2231 - ppl: 3.8331 - accu: 0.528 - ETA: 37:48 - loss: 1.2217 - ppl: 3.8265 - accu: 0.528 - ETA: 37:43 - loss: 1.2203 - ppl: 3.8202 - accu: 0.529 - ETA: 37:40 - loss: 1.2188 - ppl: 3.8135 - accu: 0.529 - ETA: 37:35 - loss: 1.2172 - ppl: 3.8068 - accu: 0.529 - ETA: 37:31 - loss: 1.2158 - ppl: 3.8005 - accu: 0.530 - ETA: 37:27 - loss: 1.2146 - ppl: 3.7945 - accu: 0.530 - ETA: 37:22 - loss: 1.2133 - ppl: 3.7885 - accu: 0.530 - ETA: 37:18 - loss: 1.2120 - ppl: 3.7825 - accu: 0.530 - ETA: 37:14 - loss: 1.2107 - ppl: 3.7766 - accu: 0.530 - ETA: 37:11 - loss: 1.2093 - ppl: 3.7703 - accu: 0.531 - ETA: 37:07 - loss: 1.2079 - ppl: 3.7642 - accu: 0.531 - ETA: 37:03 - loss: 1.2065 - ppl: 3.7582 - accu: 0.531 - ETA: 36:58 - loss: 1.2054 - ppl: 3.7527 - accu: 0.531 - ETA: 36:54 - loss: 1.2039 - ppl: 3.7465 - accu: 0.532 - ETA: 36:50 - loss: 1.2026 - ppl: 3.7406 - accu: 0.532 - ETA: 36:46 - loss: 1.2012 - ppl: 3.7346 - accu: 0.532 - ETA: 36:42 - loss: 1.1998 - ppl: 3.7287 - accu: 0.533 - ETA: 36:38 - loss: 1.1987 - ppl: 3.7233 - accu: 0.533 - ETA: 36:34 - loss: 1.1974 - ppl: 3.7176 - accu: 0.533 - ETA: 36:30 - loss: 1.1963 - ppl: 3.7124 - accu: 0.534 - ETA: 36:26 - loss: 1.1952 - ppl: 3.7072 - accu: 0.534 - ETA: 36:22 - loss: 1.1938 - ppl: 3.7013 - accu: 0.534 - ETA: 36:18 - loss: 1.1926 - ppl: 3.6960 - accu: 0.534 - ETA: 36:15 - loss: 1.1914 - ppl: 3.6907 - accu: 0.535 - ETA: 36:12 - loss: 1.1902 - ppl: 3.6854 - accu: 0.535 - ETA: 36:09 - loss: 1.1891 - ppl: 3.6804 - accu: 0.535 - ETA: 36:06 - loss: 1.1879 - ppl: 3.6750 - accu: 0.535 - ETA: 36:03 - loss: 1.1867 - ppl: 3.6698 - accu: 0.536 - ETA: 36:00 - loss: 1.1855 - ppl: 3.6645 - accu: 0.536 - ETA: 35:56 - loss: 1.1844 - ppl: 3.6595 - accu: 0.536 - ETA: 35:52 - loss: 1.1834 - ppl: 3.6548 - accu: 0.536 - ETA: 35:48 - loss: 1.1822 - ppl: 3.6496 - accu: 0.537 - ETA: 35:45 - loss: 1.1812 - ppl: 3.6450 - accu: 0.537 - ETA: 35:41 - loss: 1.1803 - ppl: 3.6405 - accu: 0.537 - ETA: 35:38 - loss: 1.1792 - ppl: 3.6358 - accu: 0.537 - ETA: 35:35 - loss: 1.1782 - ppl: 3.6310 - accu: 0.537 - ETA: 35:32 - loss: 1.1772 - ppl: 3.6264 - accu: 0.537 - ETA: 35:28 - loss: 1.1761 - ppl: 3.6216 - accu: 0.537 - ETA: 35:25 - loss: 1.1750 - ppl: 3.6169 - accu: 0.538 - ETA: 35:22 - loss: 1.1740 - ppl: 3.6124 - accu: 0.538 - ETA: 35:19 - loss: 1.1731 - ppl: 3.6082 - accu: 0.538 - ETA: 35:16 - loss: 1.1721 - ppl: 3.6035 - accu: 0.538 - ETA: 35:13 - loss: 1.1711 - ppl: 3.5991 - accu: 0.538 - ETA: 35:09 - loss: 1.1699 - ppl: 3.5942 - accu: 0.539 - ETA: 35:06 - loss: 1.1688 - ppl: 3.5895 - accu: 0.539 - ETA: 35:03 - loss: 1.1679 - ppl: 3.5854 - accu: 0.539 - ETA: 35:01 - loss: 1.1670 - ppl: 3.5812 - accu: 0.539 - ETA: 34:58 - loss: 1.1660 - ppl: 3.5768 - accu: 0.539 - ETA: 34:55 - loss: 1.1650 - ppl: 3.5723 - accu: 0.539 - ETA: 34:52 - loss: 1.1641 - ppl: 3.5682 - accu: 0.540 - ETA: 34:49 - loss: 1.1632 - ppl: 3.5642 - accu: 0.540 - ETA: 34:46 - loss: 1.1622 - ppl: 3.5598 - accu: 0.540 - ETA: 34:43 - loss: 1.1613 - ppl: 3.5557 - accu: 0.540 - ETA: 34:40 - loss: 1.1605 - ppl: 3.5519 - accu: 0.541 - ETA: 34:37 - loss: 1.1596 - ppl: 3.5479 - accu: 0.541 - ETA: 34:34 - loss: 1.1588 - ppl: 3.5440 - accu: 0.541 - ETA: 34:35 - loss: 1.1578 - ppl: 3.5398 - accu: 0.542 - ETA: 34:32 - loss: 1.1569 - ppl: 3.5359 - accu: 0.542 - ETA: 34:31 - loss: 1.1561 - ppl: 3.5321 - accu: 0.542 - ETA: 34:28 - loss: 1.1554 - ppl: 3.5285 - accu: 0.542 - ETA: 34:25 - loss: 1.1546 - ppl: 3.5249 - accu: 0.542 - ETA: 34:22 - loss: 1.1538 - ppl: 3.5213 - accu: 0.542 - ETA: 34:19 - loss: 1.1529 - ppl: 3.5174 - accu: 0.543 - ETA: 34:16 - loss: 1.1521 - ppl: 3.5137 - accu: 0.543 - ETA: 34:13 - loss: 1.1513 - ppl: 3.5100 - accu: 0.543 - ETA: 34:11 - loss: 1.1504 - ppl: 3.5061 - accu: 0.543 - ETA: 34:08 - loss: 1.1496 - ppl: 3.5025 - accu: 0.543 - ETA: 34:05 - loss: 1.1488 - ppl: 3.4988 - accu: 0.543 - ETA: 34:02 - loss: 1.1480 - ppl: 3.4952 - accu: 0.544 - ETA: 33:59 - loss: 1.1473 - ppl: 3.4918 - accu: 0.544 - ETA: 33:56 - loss: 1.1464 - ppl: 3.4880 - accu: 0.544 - ETA: 33:54 - loss: 1.1456 - ppl: 3.4844 - accu: 0.544 - ETA: 33:51 - loss: 1.1449 - ppl: 3.4812 - accu: 0.544 - ETA: 33:48 - loss: 1.1440 - ppl: 3.4775 - accu: 0.544 - ETA: 33:46 - loss: 1.1432 - ppl: 3.4739 - accu: 0.544 - ETA: 33:43 - loss: 1.1425 - ppl: 3.4705 - accu: 0.544 - ETA: 33:41 - loss: 1.1417 - ppl: 3.4670 - accu: 0.544 - ETA: 33:39 - loss: 1.1408 - ppl: 3.4633 - accu: 0.545 - ETA: 33:36 - loss: 1.1400 - ppl: 3.4598 - accu: 0.545 - ETA: 33:33 - loss: 1.1392 - ppl: 3.4563 - accu: 0.545 - ETA: 33:31 - loss: 1.1383 - ppl: 3.4527 - accu: 0.545 - ETA: 33:30 - loss: 1.1375 - ppl: 3.4493 - accu: 0.546 - ETA: 33:28 - loss: 1.1367 - ppl: 3.4457 - accu: 0.546 - ETA: 33:26 - loss: 1.1359 - ppl: 3.4424 - accu: 0.546 - ETA: 33:24 - loss: 1.1353 - ppl: 3.4394 - accu: 0.546 - ETA: 33:21 - loss: 1.1347 - ppl: 3.4364 - accu: 0.546 - ETA: 33:19 - loss: 1.1339 - ppl: 3.4332 - accu: 0.546 - ETA: 33:16 - loss: 1.1332 - ppl: 3.4301 - accu: 0.547 - ETA: 33:14 - loss: 1.1326 - ppl: 3.4270 - accu: 0.547 - ETA: 33:12 - loss: 1.1320 - ppl: 3.4241 - accu: 0.547 - ETA: 33:09 - loss: 1.1312 - ppl: 3.4209 - accu: 0.547 - ETA: 33:07 - loss: 1.1305 - ppl: 3.4177 - accu: 0.547 - ETA: 33:05 - loss: 1.1297 - ppl: 3.4143 - accu: 0.547 - ETA: 33:03 - loss: 1.1290 - ppl: 3.4112 - accu: 0.547 - ETA: 33:00 - loss: 1.1282 - ppl: 3.4079 - accu: 0.548 - ETA: 32:58 - loss: 1.1274 - ppl: 3.4046 - accu: 0.548 - ETA: 32:56 - loss: 1.1268 - ppl: 3.4017 - accu: 0.548 - ETA: 32:54 - loss: 1.1261 - ppl: 3.3989 - accu: 0.5485"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  6952/400000 [..............................] - ETA: 32:52 - loss: 1.1255 - ppl: 3.3960 - accu: 0.548 - ETA: 32:50 - loss: 1.1248 - ppl: 3.3931 - accu: 0.548 - ETA: 32:48 - loss: 1.1242 - ppl: 3.3901 - accu: 0.548 - ETA: 32:45 - loss: 1.1236 - ppl: 3.3874 - accu: 0.549 - ETA: 32:43 - loss: 1.1230 - ppl: 3.3846 - accu: 0.549 - ETA: 32:41 - loss: 1.1223 - ppl: 3.3816 - accu: 0.549 - ETA: 32:39 - loss: 1.1216 - ppl: 3.3788 - accu: 0.549 - ETA: 32:36 - loss: 1.1210 - ppl: 3.3760 - accu: 0.549 - ETA: 32:35 - loss: 1.1203 - ppl: 3.3731 - accu: 0.549 - ETA: 32:33 - loss: 1.1196 - ppl: 3.3701 - accu: 0.549 - ETA: 32:31 - loss: 1.1190 - ppl: 3.3674 - accu: 0.549 - ETA: 32:29 - loss: 1.1184 - ppl: 3.3647 - accu: 0.550 - ETA: 32:27 - loss: 1.1178 - ppl: 3.3619 - accu: 0.550 - ETA: 32:25 - loss: 1.1172 - ppl: 3.3592 - accu: 0.550 - ETA: 32:23 - loss: 1.1165 - ppl: 3.3563 - accu: 0.550 - ETA: 32:22 - loss: 1.1159 - ppl: 3.3537 - accu: 0.550 - ETA: 32:20 - loss: 1.1154 - ppl: 3.3513 - accu: 0.550 - ETA: 32:19 - loss: 1.1147 - ppl: 3.3484 - accu: 0.550 - ETA: 32:17 - loss: 1.1142 - ppl: 3.3461 - accu: 0.550 - ETA: 32:15 - loss: 1.1137 - ppl: 3.3437 - accu: 0.551 - ETA: 32:13 - loss: 1.1131 - ppl: 3.3410 - accu: 0.551 - ETA: 32:11 - loss: 1.1126 - ppl: 3.3385 - accu: 0.551 - ETA: 32:09 - loss: 1.1120 - ppl: 3.3361 - accu: 0.551 - ETA: 32:08 - loss: 1.1115 - ppl: 3.3336 - accu: 0.551 - ETA: 32:06 - loss: 1.1109 - ppl: 3.3311 - accu: 0.551 - ETA: 32:04 - loss: 1.1103 - ppl: 3.3285 - accu: 0.551 - ETA: 32:03 - loss: 1.1097 - ppl: 3.3259 - accu: 0.551 - ETA: 32:01 - loss: 1.1091 - ppl: 3.3234 - accu: 0.552 - ETA: 31:59 - loss: 1.1087 - ppl: 3.3212 - accu: 0.552 - ETA: 31:58 - loss: 1.1081 - ppl: 3.3187 - accu: 0.552 - ETA: 31:56 - loss: 1.1075 - ppl: 3.3161 - accu: 0.552 - ETA: 31:54 - loss: 1.1071 - ppl: 3.3139 - accu: 0.552 - ETA: 31:53 - loss: 1.1066 - ppl: 3.3117 - accu: 0.552 - ETA: 31:51 - loss: 1.1060 - ppl: 3.3091 - accu: 0.552 - ETA: 31:50 - loss: 1.1054 - ppl: 3.3066 - accu: 0.553 - ETA: 31:48 - loss: 1.1049 - ppl: 3.3043 - accu: 0.553 - ETA: 31:46 - loss: 1.1043 - ppl: 3.3019 - accu: 0.553 - ETA: 31:45 - loss: 1.1037 - ppl: 3.2995 - accu: 0.553 - ETA: 31:43 - loss: 1.1032 - ppl: 3.2972 - accu: 0.553 - ETA: 31:42 - loss: 1.1027 - ppl: 3.2948 - accu: 0.553 - ETA: 31:41 - loss: 1.1022 - ppl: 3.2925 - accu: 0.553 - ETA: 31:40 - loss: 1.1017 - ppl: 3.2903 - accu: 0.554 - ETA: 31:39 - loss: 1.1011 - ppl: 3.2879 - accu: 0.554 - ETA: 31:38 - loss: 1.1006 - ppl: 3.2855 - accu: 0.554 - ETA: 31:36 - loss: 1.1001 - ppl: 3.2835 - accu: 0.554 - ETA: 31:35 - loss: 1.0997 - ppl: 3.2814 - accu: 0.554 - ETA: 31:33 - loss: 1.0991 - ppl: 3.2790 - accu: 0.554 - ETA: 31:31 - loss: 1.0986 - ppl: 3.2767 - accu: 0.554 - ETA: 31:30 - loss: 1.0980 - ppl: 3.2745 - accu: 0.554 - ETA: 31:28 - loss: 1.0975 - ppl: 3.2722 - accu: 0.554 - ETA: 31:26 - loss: 1.0970 - ppl: 3.2700 - accu: 0.554 - ETA: 31:24 - loss: 1.0964 - ppl: 3.2676 - accu: 0.554 - ETA: 31:23 - loss: 1.0958 - ppl: 3.2652 - accu: 0.554 - ETA: 31:21 - loss: 1.0953 - ppl: 3.2630 - accu: 0.554 - ETA: 31:20 - loss: 1.0948 - ppl: 3.2609 - accu: 0.555 - ETA: 31:18 - loss: 1.0944 - ppl: 3.2588 - accu: 0.555 - ETA: 31:17 - loss: 1.0939 - ppl: 3.2567 - accu: 0.555 - ETA: 31:15 - loss: 1.0935 - ppl: 3.2549 - accu: 0.555 - ETA: 31:14 - loss: 1.0931 - ppl: 3.2530 - accu: 0.555 - ETA: 31:13 - loss: 1.0926 - ppl: 3.2508 - accu: 0.555 - ETA: 31:11 - loss: 1.0921 - ppl: 3.2487 - accu: 0.555 - ETA: 31:10 - loss: 1.0916 - ppl: 3.2465 - accu: 0.555 - ETA: 31:09 - loss: 1.0911 - ppl: 3.2444 - accu: 0.555 - ETA: 31:08 - loss: 1.0906 - ppl: 3.2423 - accu: 0.555 - ETA: 31:06 - loss: 1.0902 - ppl: 3.2404 - accu: 0.555 - ETA: 31:05 - loss: 1.0897 - ppl: 3.2384 - accu: 0.555 - ETA: 31:03 - loss: 1.0893 - ppl: 3.2363 - accu: 0.556 - ETA: 31:02 - loss: 1.0888 - ppl: 3.2344 - accu: 0.556 - ETA: 31:01 - loss: 1.0884 - ppl: 3.2324 - accu: 0.556 - ETA: 30:59 - loss: 1.0880 - ppl: 3.2306 - accu: 0.556 - ETA: 30:58 - loss: 1.0875 - ppl: 3.2286 - accu: 0.556 - ETA: 30:57 - loss: 1.0872 - ppl: 3.2269 - accu: 0.556 - ETA: 30:56 - loss: 1.0867 - ppl: 3.2250 - accu: 0.556 - ETA: 30:54 - loss: 1.0862 - ppl: 3.2228 - accu: 0.556 - ETA: 30:52 - loss: 1.0857 - ppl: 3.2207 - accu: 0.556 - ETA: 30:51 - loss: 1.0852 - ppl: 3.2188 - accu: 0.557 - ETA: 30:50 - loss: 1.0849 - ppl: 3.2171 - accu: 0.557 - ETA: 30:48 - loss: 1.0845 - ppl: 3.2154 - accu: 0.557 - ETA: 30:47 - loss: 1.0840 - ppl: 3.2134 - accu: 0.557 - ETA: 30:46 - loss: 1.0836 - ppl: 3.2116 - accu: 0.557 - ETA: 30:44 - loss: 1.0832 - ppl: 3.2097 - accu: 0.557 - ETA: 30:43 - loss: 1.0828 - ppl: 3.2079 - accu: 0.557 - ETA: 30:41 - loss: 1.0824 - ppl: 3.2061 - accu: 0.557 - ETA: 30:40 - loss: 1.0820 - ppl: 3.2043 - accu: 0.557 - ETA: 30:39 - loss: 1.0815 - ppl: 3.2024 - accu: 0.557 - ETA: 30:38 - loss: 1.0811 - ppl: 3.2004 - accu: 0.557 - ETA: 30:36 - loss: 1.0805 - ppl: 3.1984 - accu: 0.557 - ETA: 30:35 - loss: 1.0800 - ppl: 3.1964 - accu: 0.557 - ETA: 30:33 - loss: 1.0796 - ppl: 3.1945 - accu: 0.557 - ETA: 30:32 - loss: 1.0792 - ppl: 3.1927 - accu: 0.558 - ETA: 30:31 - loss: 1.0788 - ppl: 3.1910 - accu: 0.558 - ETA: 30:30 - loss: 1.0784 - ppl: 3.1892 - accu: 0.558 - ETA: 30:29 - loss: 1.0779 - ppl: 3.1874 - accu: 0.558 - ETA: 30:28 - loss: 1.0775 - ppl: 3.1855 - accu: 0.558 - ETA: 30:26 - loss: 1.0770 - ppl: 3.1836 - accu: 0.558 - ETA: 30:26 - loss: 1.0766 - ppl: 3.1819 - accu: 0.558 - ETA: 30:25 - loss: 1.0762 - ppl: 3.1801 - accu: 0.558 - ETA: 30:25 - loss: 1.0757 - ppl: 3.1782 - accu: 0.559 - ETA: 30:23 - loss: 1.0754 - ppl: 3.1766 - accu: 0.559 - ETA: 30:22 - loss: 1.0749 - ppl: 3.1746 - accu: 0.559 - ETA: 30:21 - loss: 1.0744 - ppl: 3.1728 - accu: 0.559 - ETA: 30:20 - loss: 1.0741 - ppl: 3.1712 - accu: 0.559 - ETA: 30:18 - loss: 1.0737 - ppl: 3.1697 - accu: 0.559 - ETA: 30:17 - loss: 1.0733 - ppl: 3.1678 - accu: 0.559 - ETA: 30:16 - loss: 1.0729 - ppl: 3.1661 - accu: 0.559 - ETA: 30:15 - loss: 1.0724 - ppl: 3.1643 - accu: 0.560 - ETA: 30:13 - loss: 1.0721 - ppl: 3.1628 - accu: 0.560 - ETA: 30:12 - loss: 1.0717 - ppl: 3.1610 - accu: 0.560 - ETA: 30:11 - loss: 1.0712 - ppl: 3.1592 - accu: 0.560 - ETA: 30:10 - loss: 1.0709 - ppl: 3.1576 - accu: 0.560 - ETA: 30:08 - loss: 1.0704 - ppl: 3.1558 - accu: 0.560 - ETA: 30:07 - loss: 1.0699 - ppl: 3.1540 - accu: 0.560 - ETA: 30:06 - loss: 1.0695 - ppl: 3.1523 - accu: 0.560 - ETA: 30:05 - loss: 1.0691 - ppl: 3.1506 - accu: 0.561 - ETA: 30:04 - loss: 1.0687 - ppl: 3.1489 - accu: 0.561 - ETA: 30:03 - loss: 1.0683 - ppl: 3.1471 - accu: 0.561 - ETA: 30:02 - loss: 1.0678 - ppl: 3.1453 - accu: 0.561 - ETA: 30:01 - loss: 1.0673 - ppl: 3.1435 - accu: 0.561 - ETA: 30:00 - loss: 1.0669 - ppl: 3.1418 - accu: 0.561 - ETA: 29:58 - loss: 1.0665 - ppl: 3.1402 - accu: 0.561 - ETA: 29:57 - loss: 1.0662 - ppl: 3.1388 - accu: 0.561 - ETA: 29:57 - loss: 1.0658 - ppl: 3.1372 - accu: 0.561 - ETA: 29:55 - loss: 1.0654 - ppl: 3.1354 - accu: 0.561 - ETA: 29:54 - loss: 1.0649 - ppl: 3.1335 - accu: 0.562 - ETA: 29:53 - loss: 1.0644 - ppl: 3.1317 - accu: 0.562 - ETA: 29:52 - loss: 1.0639 - ppl: 3.1299 - accu: 0.562 - ETA: 29:51 - loss: 1.0636 - ppl: 3.1285 - accu: 0.562 - ETA: 29:50 - loss: 1.0632 - ppl: 3.1268 - accu: 0.562 - ETA: 29:49 - loss: 1.0628 - ppl: 3.1253 - accu: 0.562 - ETA: 29:48 - loss: 1.0625 - ppl: 3.1238 - accu: 0.562 - ETA: 29:46 - loss: 1.0621 - ppl: 3.1222 - accu: 0.563 - ETA: 29:45 - loss: 1.0616 - ppl: 3.1205 - accu: 0.563 - ETA: 29:45 - loss: 1.0612 - ppl: 3.1187 - accu: 0.563 - ETA: 29:44 - loss: 1.0608 - ppl: 3.1171 - accu: 0.563 - ETA: 29:42 - loss: 1.0604 - ppl: 3.1155 - accu: 0.563 - ETA: 29:41 - loss: 1.0599 - ppl: 3.1138 - accu: 0.563 - ETA: 29:40 - loss: 1.0595 - ppl: 3.1121 - accu: 0.563 - ETA: 29:39 - loss: 1.0590 - ppl: 3.1103 - accu: 0.564 - ETA: 29:38 - loss: 1.0586 - ppl: 3.1087 - accu: 0.564 - ETA: 29:37 - loss: 1.0582 - ppl: 3.1072 - accu: 0.564 - ETA: 29:36 - loss: 1.0578 - ppl: 3.1056 - accu: 0.564 - ETA: 29:35 - loss: 1.0573 - ppl: 3.1038 - accu: 0.564 - ETA: 29:34 - loss: 1.0570 - ppl: 3.1024 - accu: 0.564 - ETA: 29:33 - loss: 1.0567 - ppl: 3.1010 - accu: 0.565 - ETA: 29:32 - loss: 1.0563 - ppl: 3.0994 - accu: 0.565 - ETA: 29:31 - loss: 1.0558 - ppl: 3.0978 - accu: 0.5653"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  9288/400000 [..............................] - ETA: 29:30 - loss: 1.0555 - ppl: 3.0963 - accu: 0.565 - ETA: 29:29 - loss: 1.0551 - ppl: 3.0948 - accu: 0.565 - ETA: 29:28 - loss: 1.0548 - ppl: 3.0934 - accu: 0.565 - ETA: 29:28 - loss: 1.0544 - ppl: 3.0919 - accu: 0.565 - ETA: 29:26 - loss: 1.0540 - ppl: 3.0903 - accu: 0.565 - ETA: 29:26 - loss: 1.0537 - ppl: 3.0889 - accu: 0.565 - ETA: 29:25 - loss: 1.0533 - ppl: 3.0875 - accu: 0.566 - ETA: 29:25 - loss: 1.0529 - ppl: 3.0859 - accu: 0.566 - ETA: 29:24 - loss: 1.0525 - ppl: 3.0844 - accu: 0.566 - ETA: 29:23 - loss: 1.0521 - ppl: 3.0828 - accu: 0.566 - ETA: 29:22 - loss: 1.0516 - ppl: 3.0812 - accu: 0.566 - ETA: 29:21 - loss: 1.0513 - ppl: 3.0798 - accu: 0.566 - ETA: 29:20 - loss: 1.0509 - ppl: 3.0783 - accu: 0.566 - ETA: 29:19 - loss: 1.0505 - ppl: 3.0768 - accu: 0.567 - ETA: 29:18 - loss: 1.0502 - ppl: 3.0754 - accu: 0.567 - ETA: 29:17 - loss: 1.0499 - ppl: 3.0741 - accu: 0.567 - ETA: 29:17 - loss: 1.0495 - ppl: 3.0727 - accu: 0.567 - ETA: 29:16 - loss: 1.0491 - ppl: 3.0712 - accu: 0.567 - ETA: 29:15 - loss: 1.0487 - ppl: 3.0696 - accu: 0.567 - ETA: 29:14 - loss: 1.0484 - ppl: 3.0684 - accu: 0.567 - ETA: 29:13 - loss: 1.0480 - ppl: 3.0668 - accu: 0.567 - ETA: 29:12 - loss: 1.0477 - ppl: 3.0655 - accu: 0.568 - ETA: 29:11 - loss: 1.0473 - ppl: 3.0640 - accu: 0.568 - ETA: 29:11 - loss: 1.0470 - ppl: 3.0627 - accu: 0.568 - ETA: 29:10 - loss: 1.0467 - ppl: 3.0615 - accu: 0.568 - ETA: 29:09 - loss: 1.0463 - ppl: 3.0601 - accu: 0.568 - ETA: 29:07 - loss: 1.0460 - ppl: 3.0588 - accu: 0.568 - ETA: 29:06 - loss: 1.0456 - ppl: 3.0573 - accu: 0.568 - ETA: 29:05 - loss: 1.0454 - ppl: 3.0562 - accu: 0.568 - ETA: 29:05 - loss: 1.0450 - ppl: 3.0548 - accu: 0.568 - ETA: 29:04 - loss: 1.0447 - ppl: 3.0534 - accu: 0.568 - ETA: 29:03 - loss: 1.0444 - ppl: 3.0521 - accu: 0.569 - ETA: 29:02 - loss: 1.0440 - ppl: 3.0507 - accu: 0.569 - ETA: 29:01 - loss: 1.0436 - ppl: 3.0493 - accu: 0.569 - ETA: 29:00 - loss: 1.0433 - ppl: 3.0480 - accu: 0.569 - ETA: 28:59 - loss: 1.0429 - ppl: 3.0466 - accu: 0.569 - ETA: 28:59 - loss: 1.0425 - ppl: 3.0451 - accu: 0.569 - ETA: 28:58 - loss: 1.0421 - ppl: 3.0436 - accu: 0.569 - ETA: 28:57 - loss: 1.0417 - ppl: 3.0421 - accu: 0.569 - ETA: 28:56 - loss: 1.0414 - ppl: 3.0409 - accu: 0.570 - ETA: 28:55 - loss: 1.0411 - ppl: 3.0396 - accu: 0.570 - ETA: 28:54 - loss: 1.0407 - ppl: 3.0383 - accu: 0.570 - ETA: 28:54 - loss: 1.0404 - ppl: 3.0369 - accu: 0.570 - ETA: 28:53 - loss: 1.0400 - ppl: 3.0354 - accu: 0.570 - ETA: 28:52 - loss: 1.0396 - ppl: 3.0342 - accu: 0.570 - ETA: 28:51 - loss: 1.0393 - ppl: 3.0329 - accu: 0.570 - ETA: 28:50 - loss: 1.0389 - ppl: 3.0315 - accu: 0.570 - ETA: 28:50 - loss: 1.0385 - ppl: 3.0300 - accu: 0.571 - ETA: 28:49 - loss: 1.0382 - ppl: 3.0287 - accu: 0.571 - ETA: 28:48 - loss: 1.0378 - ppl: 3.0273 - accu: 0.571 - ETA: 28:47 - loss: 1.0375 - ppl: 3.0259 - accu: 0.571 - ETA: 28:46 - loss: 1.0371 - ppl: 3.0246 - accu: 0.571 - ETA: 28:46 - loss: 1.0368 - ppl: 3.0233 - accu: 0.571 - ETA: 28:45 - loss: 1.0364 - ppl: 3.0219 - accu: 0.571 - ETA: 28:44 - loss: 1.0360 - ppl: 3.0205 - accu: 0.572 - ETA: 28:43 - loss: 1.0356 - ppl: 3.0192 - accu: 0.572 - ETA: 28:42 - loss: 1.0354 - ppl: 3.0181 - accu: 0.572 - ETA: 28:42 - loss: 1.0350 - ppl: 3.0167 - accu: 0.572 - ETA: 28:41 - loss: 1.0346 - ppl: 3.0152 - accu: 0.572 - ETA: 28:40 - loss: 1.0343 - ppl: 3.0141 - accu: 0.572 - ETA: 28:40 - loss: 1.0338 - ppl: 3.0125 - accu: 0.572 - ETA: 28:39 - loss: 1.0335 - ppl: 3.0112 - accu: 0.573 - ETA: 28:38 - loss: 1.0332 - ppl: 3.0100 - accu: 0.573 - ETA: 28:38 - loss: 1.0329 - ppl: 3.0088 - accu: 0.573 - ETA: 28:37 - loss: 1.0325 - ppl: 3.0075 - accu: 0.573 - ETA: 28:37 - loss: 1.0323 - ppl: 3.0065 - accu: 0.573 - ETA: 28:36 - loss: 1.0319 - ppl: 3.0051 - accu: 0.573 - ETA: 28:36 - loss: 1.0317 - ppl: 3.0041 - accu: 0.573 - ETA: 28:36 - loss: 1.0314 - ppl: 3.0029 - accu: 0.573 - ETA: 28:35 - loss: 1.0310 - ppl: 3.0016 - accu: 0.573 - ETA: 28:34 - loss: 1.0307 - ppl: 3.0005 - accu: 0.573 - ETA: 28:34 - loss: 1.0304 - ppl: 2.9992 - accu: 0.574 - ETA: 28:33 - loss: 1.0300 - ppl: 2.9979 - accu: 0.574 - ETA: 28:33 - loss: 1.0297 - ppl: 2.9967 - accu: 0.574 - ETA: 28:33 - loss: 1.0293 - ppl: 2.9954 - accu: 0.574 - ETA: 28:32 - loss: 1.0290 - ppl: 2.9941 - accu: 0.574 - ETA: 28:32 - loss: 1.0286 - ppl: 2.9926 - accu: 0.574 - ETA: 28:31 - loss: 1.0282 - ppl: 2.9912 - accu: 0.574 - ETA: 28:31 - loss: 1.0279 - ppl: 2.9900 - accu: 0.575 - ETA: 28:30 - loss: 1.0275 - ppl: 2.9888 - accu: 0.575 - ETA: 28:30 - loss: 1.0272 - ppl: 2.9876 - accu: 0.575 - ETA: 28:29 - loss: 1.0268 - ppl: 2.9862 - accu: 0.575 - ETA: 28:29 - loss: 1.0264 - ppl: 2.9849 - accu: 0.575 - ETA: 28:28 - loss: 1.0261 - ppl: 2.9837 - accu: 0.575 - ETA: 28:27 - loss: 1.0258 - ppl: 2.9824 - accu: 0.576 - ETA: 28:27 - loss: 1.0255 - ppl: 2.9813 - accu: 0.576 - ETA: 28:26 - loss: 1.0252 - ppl: 2.9802 - accu: 0.576 - ETA: 28:25 - loss: 1.0248 - ppl: 2.9790 - accu: 0.576 - ETA: 28:25 - loss: 1.0245 - ppl: 2.9777 - accu: 0.576 - ETA: 28:24 - loss: 1.0242 - ppl: 2.9767 - accu: 0.576 - ETA: 28:24 - loss: 1.0239 - ppl: 2.9755 - accu: 0.576 - ETA: 28:23 - loss: 1.0236 - ppl: 2.9744 - accu: 0.576 - ETA: 28:22 - loss: 1.0233 - ppl: 2.9732 - accu: 0.576 - ETA: 28:22 - loss: 1.0230 - ppl: 2.9720 - accu: 0.576 - ETA: 28:21 - loss: 1.0226 - ppl: 2.9707 - accu: 0.577 - ETA: 28:20 - loss: 1.0223 - ppl: 2.9696 - accu: 0.577 - ETA: 28:20 - loss: 1.0220 - ppl: 2.9685 - accu: 0.577 - ETA: 28:19 - loss: 1.0218 - ppl: 2.9675 - accu: 0.577 - ETA: 28:18 - loss: 1.0214 - ppl: 2.9662 - accu: 0.577 - ETA: 28:18 - loss: 1.0212 - ppl: 2.9653 - accu: 0.577 - ETA: 28:17 - loss: 1.0209 - ppl: 2.9642 - accu: 0.577 - ETA: 28:16 - loss: 1.0206 - ppl: 2.9630 - accu: 0.577 - ETA: 28:15 - loss: 1.0202 - ppl: 2.9617 - accu: 0.577 - ETA: 28:15 - loss: 1.0199 - ppl: 2.9606 - accu: 0.578 - ETA: 28:14 - loss: 1.0196 - ppl: 2.9595 - accu: 0.578 - ETA: 28:14 - loss: 1.0193 - ppl: 2.9583 - accu: 0.578 - ETA: 28:13 - loss: 1.0190 - ppl: 2.9573 - accu: 0.578 - ETA: 28:13 - loss: 1.0187 - ppl: 2.9561 - accu: 0.578 - ETA: 28:13 - loss: 1.0183 - ppl: 2.9548 - accu: 0.578 - ETA: 28:12 - loss: 1.0180 - ppl: 2.9537 - accu: 0.578 - ETA: 28:11 - loss: 1.0177 - ppl: 2.9525 - accu: 0.578 - ETA: 28:11 - loss: 1.0173 - ppl: 2.9513 - accu: 0.579 - ETA: 28:10 - loss: 1.0170 - ppl: 2.9501 - accu: 0.579 - ETA: 28:09 - loss: 1.0166 - ppl: 2.9489 - accu: 0.579 - ETA: 28:09 - loss: 1.0163 - ppl: 2.9476 - accu: 0.579 - ETA: 28:08 - loss: 1.0160 - ppl: 2.9466 - accu: 0.579 - ETA: 28:08 - loss: 1.0157 - ppl: 2.9456 - accu: 0.579 - ETA: 28:07 - loss: 1.0155 - ppl: 2.9446 - accu: 0.579 - ETA: 28:07 - loss: 1.0152 - ppl: 2.9435 - accu: 0.579 - ETA: 28:07 - loss: 1.0149 - ppl: 2.9425 - accu: 0.580 - ETA: 28:07 - loss: 1.0145 - ppl: 2.9413 - accu: 0.580 - ETA: 28:06 - loss: 1.0142 - ppl: 2.9402 - accu: 0.580 - ETA: 28:05 - loss: 1.0140 - ppl: 2.9393 - accu: 0.580 - ETA: 28:05 - loss: 1.0138 - ppl: 2.9384 - accu: 0.580 - ETA: 28:04 - loss: 1.0136 - ppl: 2.9374 - accu: 0.580 - ETA: 28:04 - loss: 1.0132 - ppl: 2.9363 - accu: 0.580 - ETA: 28:03 - loss: 1.0130 - ppl: 2.9353 - accu: 0.580 - ETA: 28:03 - loss: 1.0127 - ppl: 2.9343 - accu: 0.580 - ETA: 28:02 - loss: 1.0124 - ppl: 2.9332 - accu: 0.580 - ETA: 28:02 - loss: 1.0121 - ppl: 2.9322 - accu: 0.581 - ETA: 28:01 - loss: 1.0118 - ppl: 2.9310 - accu: 0.581 - ETA: 28:00 - loss: 1.0115 - ppl: 2.9300 - accu: 0.581 - ETA: 28:00 - loss: 1.0113 - ppl: 2.9291 - accu: 0.581 - ETA: 27:59 - loss: 1.0111 - ppl: 2.9282 - accu: 0.581 - ETA: 27:59 - loss: 1.0108 - ppl: 2.9272 - accu: 0.581 - ETA: 27:58 - loss: 1.0105 - ppl: 2.9263 - accu: 0.581 - ETA: 27:58 - loss: 1.0102 - ppl: 2.9251 - accu: 0.581 - ETA: 27:57 - loss: 1.0099 - ppl: 2.9239 - accu: 0.582 - ETA: 27:56 - loss: 1.0095 - ppl: 2.9228 - accu: 0.582 - ETA: 27:56 - loss: 1.0093 - ppl: 2.9218 - accu: 0.582 - ETA: 27:55 - loss: 1.0090 - ppl: 2.9207 - accu: 0.582 - ETA: 27:55 - loss: 1.0087 - ppl: 2.9197 - accu: 0.582 - ETA: 27:54 - loss: 1.0084 - ppl: 2.9187 - accu: 0.582 - ETA: 27:53 - loss: 1.0081 - ppl: 2.9176 - accu: 0.582 - ETA: 27:53 - loss: 1.0079 - ppl: 2.9167 - accu: 0.582 - ETA: 27:52 - loss: 1.0075 - ppl: 2.9156 - accu: 0.5829"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 11624/400000 [..............................] - ETA: 27:52 - loss: 1.0072 - ppl: 2.9146 - accu: 0.583 - ETA: 27:51 - loss: 1.0070 - ppl: 2.9136 - accu: 0.583 - ETA: 27:51 - loss: 1.0067 - ppl: 2.9127 - accu: 0.583 - ETA: 27:50 - loss: 1.0065 - ppl: 2.9117 - accu: 0.583 - ETA: 27:50 - loss: 1.0062 - ppl: 2.9107 - accu: 0.583 - ETA: 27:49 - loss: 1.0059 - ppl: 2.9096 - accu: 0.583 - ETA: 27:49 - loss: 1.0056 - ppl: 2.9086 - accu: 0.583 - ETA: 27:49 - loss: 1.0053 - ppl: 2.9077 - accu: 0.583 - ETA: 27:49 - loss: 1.0051 - ppl: 2.9067 - accu: 0.583 - ETA: 27:48 - loss: 1.0048 - ppl: 2.9058 - accu: 0.584 - ETA: 27:48 - loss: 1.0045 - ppl: 2.9048 - accu: 0.584 - ETA: 27:48 - loss: 1.0043 - ppl: 2.9039 - accu: 0.584 - ETA: 27:47 - loss: 1.0041 - ppl: 2.9030 - accu: 0.584 - ETA: 27:47 - loss: 1.0038 - ppl: 2.9021 - accu: 0.584 - ETA: 27:46 - loss: 1.0036 - ppl: 2.9012 - accu: 0.584 - ETA: 27:46 - loss: 1.0033 - ppl: 2.9003 - accu: 0.584 - ETA: 27:45 - loss: 1.0030 - ppl: 2.8992 - accu: 0.584 - ETA: 27:44 - loss: 1.0028 - ppl: 2.8982 - accu: 0.584 - ETA: 27:44 - loss: 1.0025 - ppl: 2.8973 - accu: 0.584 - ETA: 27:43 - loss: 1.0022 - ppl: 2.8963 - accu: 0.584 - ETA: 27:42 - loss: 1.0019 - ppl: 2.8952 - accu: 0.584 - ETA: 27:42 - loss: 1.0017 - ppl: 2.8944 - accu: 0.585 - ETA: 27:41 - loss: 1.0014 - ppl: 2.8934 - accu: 0.585 - ETA: 27:41 - loss: 1.0011 - ppl: 2.8924 - accu: 0.585 - ETA: 27:40 - loss: 1.0009 - ppl: 2.8915 - accu: 0.585 - ETA: 27:39 - loss: 1.0007 - ppl: 2.8908 - accu: 0.585 - ETA: 27:39 - loss: 1.0004 - ppl: 2.8899 - accu: 0.585 - ETA: 27:39 - loss: 1.0001 - ppl: 2.8888 - accu: 0.585 - ETA: 27:39 - loss: 1.0000 - ppl: 2.8881 - accu: 0.585 - ETA: 27:38 - loss: 0.9997 - ppl: 2.8873 - accu: 0.585 - ETA: 27:37 - loss: 0.9995 - ppl: 2.8863 - accu: 0.585 - ETA: 27:37 - loss: 0.9991 - ppl: 2.8852 - accu: 0.586 - ETA: 27:36 - loss: 0.9989 - ppl: 2.8843 - accu: 0.586 - ETA: 27:36 - loss: 0.9987 - ppl: 2.8835 - accu: 0.586 - ETA: 27:35 - loss: 0.9984 - ppl: 2.8824 - accu: 0.586 - ETA: 27:35 - loss: 0.9980 - ppl: 2.8813 - accu: 0.586 - ETA: 27:35 - loss: 0.9978 - ppl: 2.8804 - accu: 0.586 - ETA: 27:34 - loss: 0.9976 - ppl: 2.8796 - accu: 0.586 - ETA: 27:34 - loss: 0.9973 - ppl: 2.8787 - accu: 0.586 - ETA: 27:33 - loss: 0.9970 - ppl: 2.8778 - accu: 0.586 - ETA: 27:33 - loss: 0.9967 - ppl: 2.8768 - accu: 0.586 - ETA: 27:32 - loss: 0.9965 - ppl: 2.8760 - accu: 0.586 - ETA: 27:31 - loss: 0.9962 - ppl: 2.8749 - accu: 0.587 - ETA: 27:31 - loss: 0.9959 - ppl: 2.8740 - accu: 0.587 - ETA: 27:31 - loss: 0.9957 - ppl: 2.8731 - accu: 0.587 - ETA: 27:30 - loss: 0.9954 - ppl: 2.8722 - accu: 0.587 - ETA: 27:30 - loss: 0.9951 - ppl: 2.8710 - accu: 0.587 - ETA: 27:29 - loss: 0.9948 - ppl: 2.8702 - accu: 0.587 - ETA: 27:29 - loss: 0.9945 - ppl: 2.8692 - accu: 0.587 - ETA: 27:28 - loss: 0.9943 - ppl: 2.8683 - accu: 0.588 - ETA: 27:28 - loss: 0.9939 - ppl: 2.8672 - accu: 0.588 - ETA: 27:27 - loss: 0.9937 - ppl: 2.8665 - accu: 0.588 - ETA: 27:27 - loss: 0.9936 - ppl: 2.8657 - accu: 0.588 - ETA: 27:26 - loss: 0.9933 - ppl: 2.8647 - accu: 0.588 - ETA: 27:26 - loss: 0.9931 - ppl: 2.8640 - accu: 0.588 - ETA: 27:25 - loss: 0.9928 - ppl: 2.8632 - accu: 0.588 - ETA: 27:25 - loss: 0.9926 - ppl: 2.8623 - accu: 0.588 - ETA: 27:24 - loss: 0.9923 - ppl: 2.8613 - accu: 0.588 - ETA: 27:23 - loss: 0.9920 - ppl: 2.8603 - accu: 0.589 - ETA: 27:23 - loss: 0.9917 - ppl: 2.8595 - accu: 0.589 - ETA: 27:22 - loss: 0.9915 - ppl: 2.8585 - accu: 0.589 - ETA: 27:22 - loss: 0.9912 - ppl: 2.8577 - accu: 0.589 - ETA: 27:21 - loss: 0.9910 - ppl: 2.8568 - accu: 0.589 - ETA: 27:21 - loss: 0.9907 - ppl: 2.8559 - accu: 0.589 - ETA: 27:20 - loss: 0.9905 - ppl: 2.8551 - accu: 0.589 - ETA: 27:20 - loss: 0.9903 - ppl: 2.8543 - accu: 0.589 - ETA: 27:19 - loss: 0.9900 - ppl: 2.8534 - accu: 0.589 - ETA: 27:19 - loss: 0.9898 - ppl: 2.8525 - accu: 0.590 - ETA: 27:18 - loss: 0.9895 - ppl: 2.8517 - accu: 0.590 - ETA: 27:17 - loss: 0.9892 - ppl: 2.8507 - accu: 0.590 - ETA: 27:17 - loss: 0.9889 - ppl: 2.8497 - accu: 0.590 - ETA: 27:16 - loss: 0.9887 - ppl: 2.8489 - accu: 0.590 - ETA: 27:16 - loss: 0.9884 - ppl: 2.8480 - accu: 0.590 - ETA: 27:16 - loss: 0.9882 - ppl: 2.8471 - accu: 0.590 - ETA: 27:15 - loss: 0.9879 - ppl: 2.8463 - accu: 0.590 - ETA: 27:14 - loss: 0.9877 - ppl: 2.8456 - accu: 0.591 - ETA: 27:14 - loss: 0.9875 - ppl: 2.8447 - accu: 0.591 - ETA: 27:13 - loss: 0.9873 - ppl: 2.8439 - accu: 0.591 - ETA: 27:13 - loss: 0.9870 - ppl: 2.8430 - accu: 0.591 - ETA: 27:13 - loss: 0.9867 - ppl: 2.8420 - accu: 0.591 - ETA: 27:13 - loss: 0.9865 - ppl: 2.8412 - accu: 0.591 - ETA: 27:12 - loss: 0.9862 - ppl: 2.8402 - accu: 0.591 - ETA: 27:12 - loss: 0.9859 - ppl: 2.8394 - accu: 0.591 - ETA: 27:12 - loss: 0.9857 - ppl: 2.8385 - accu: 0.592 - ETA: 27:12 - loss: 0.9854 - ppl: 2.8375 - accu: 0.592 - ETA: 27:11 - loss: 0.9851 - ppl: 2.8365 - accu: 0.592 - ETA: 27:11 - loss: 0.9848 - ppl: 2.8357 - accu: 0.592 - ETA: 27:10 - loss: 0.9846 - ppl: 2.8349 - accu: 0.592 - ETA: 27:10 - loss: 0.9844 - ppl: 2.8342 - accu: 0.592 - ETA: 27:09 - loss: 0.9842 - ppl: 2.8334 - accu: 0.592 - ETA: 27:09 - loss: 0.9839 - ppl: 2.8326 - accu: 0.592 - ETA: 27:08 - loss: 0.9837 - ppl: 2.8318 - accu: 0.592 - ETA: 27:08 - loss: 0.9834 - ppl: 2.8309 - accu: 0.593 - ETA: 27:08 - loss: 0.9832 - ppl: 2.8302 - accu: 0.593 - ETA: 27:07 - loss: 0.9830 - ppl: 2.8293 - accu: 0.593 - ETA: 27:07 - loss: 0.9827 - ppl: 2.8284 - accu: 0.593 - ETA: 27:07 - loss: 0.9825 - ppl: 2.8276 - accu: 0.593 - ETA: 27:06 - loss: 0.9823 - ppl: 2.8269 - accu: 0.593 - ETA: 27:06 - loss: 0.9820 - ppl: 2.8259 - accu: 0.593 - ETA: 27:06 - loss: 0.9816 - ppl: 2.8249 - accu: 0.594 - ETA: 27:05 - loss: 0.9814 - ppl: 2.8241 - accu: 0.594 - ETA: 27:05 - loss: 0.9812 - ppl: 2.8233 - accu: 0.594 - ETA: 27:05 - loss: 0.9810 - ppl: 2.8225 - accu: 0.594 - ETA: 27:04 - loss: 0.9807 - ppl: 2.8218 - accu: 0.594 - ETA: 27:04 - loss: 0.9806 - ppl: 2.8212 - accu: 0.594 - ETA: 27:04 - loss: 0.9804 - ppl: 2.8204 - accu: 0.594 - ETA: 27:03 - loss: 0.9801 - ppl: 2.8195 - accu: 0.594 - ETA: 27:03 - loss: 0.9799 - ppl: 2.8187 - accu: 0.594 - ETA: 27:03 - loss: 0.9797 - ppl: 2.8180 - accu: 0.594 - ETA: 27:02 - loss: 0.9794 - ppl: 2.8171 - accu: 0.594 - ETA: 27:02 - loss: 0.9792 - ppl: 2.8164 - accu: 0.595 - ETA: 27:02 - loss: 0.9790 - ppl: 2.8157 - accu: 0.595 - ETA: 27:01 - loss: 0.9787 - ppl: 2.8148 - accu: 0.595 - ETA: 27:01 - loss: 0.9785 - ppl: 2.8140 - accu: 0.595 - ETA: 27:01 - loss: 0.9782 - ppl: 2.8131 - accu: 0.595 - ETA: 27:01 - loss: 0.9780 - ppl: 2.8123 - accu: 0.595 - ETA: 27:00 - loss: 0.9776 - ppl: 2.8112 - accu: 0.595 - ETA: 27:00 - loss: 0.9774 - ppl: 2.8105 - accu: 0.595 - ETA: 27:00 - loss: 0.9772 - ppl: 2.8098 - accu: 0.595 - ETA: 26:59 - loss: 0.9769 - ppl: 2.8090 - accu: 0.595 - ETA: 26:59 - loss: 0.9769 - ppl: 2.8086 - accu: 0.595 - ETA: 26:58 - loss: 0.9767 - ppl: 2.8079 - accu: 0.595 - ETA: 26:58 - loss: 0.9765 - ppl: 2.8072 - accu: 0.596 - ETA: 26:58 - loss: 0.9763 - ppl: 2.8066 - accu: 0.596 - ETA: 26:57 - loss: 0.9762 - ppl: 2.8061 - accu: 0.596 - ETA: 26:57 - loss: 0.9761 - ppl: 2.8055 - accu: 0.596 - ETA: 26:56 - loss: 0.9758 - ppl: 2.8046 - accu: 0.596 - ETA: 26:56 - loss: 0.9756 - ppl: 2.8038 - accu: 0.596 - ETA: 26:55 - loss: 0.9753 - ppl: 2.8030 - accu: 0.596 - ETA: 26:55 - loss: 0.9752 - ppl: 2.8024 - accu: 0.596 - ETA: 26:55 - loss: 0.9750 - ppl: 2.8017 - accu: 0.596 - ETA: 26:54 - loss: 0.9747 - ppl: 2.8010 - accu: 0.596 - ETA: 26:54 - loss: 0.9745 - ppl: 2.8002 - accu: 0.596 - ETA: 26:53 - loss: 0.9743 - ppl: 2.7993 - accu: 0.596 - ETA: 26:53 - loss: 0.9740 - ppl: 2.7986 - accu: 0.596 - ETA: 26:52 - loss: 0.9738 - ppl: 2.7979 - accu: 0.596 - ETA: 26:52 - loss: 0.9736 - ppl: 2.7971 - accu: 0.597 - ETA: 26:51 - loss: 0.9734 - ppl: 2.7963 - accu: 0.597 - ETA: 26:51 - loss: 0.9731 - ppl: 2.7955 - accu: 0.597 - ETA: 26:51 - loss: 0.9729 - ppl: 2.7948 - accu: 0.597 - ETA: 26:51 - loss: 0.9727 - ppl: 2.7940 - accu: 0.597 - ETA: 26:51 - loss: 0.9725 - ppl: 2.7933 - accu: 0.597 - ETA: 26:50 - loss: 0.9722 - ppl: 2.7925 - accu: 0.597 - ETA: 26:50 - loss: 0.9720 - ppl: 2.7918 - accu: 0.597 - ETA: 26:49 - loss: 0.9717 - ppl: 2.7910 - accu: 0.597 - ETA: 26:49 - loss: 0.9715 - ppl: 2.7902 - accu: 0.5977"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 13960/400000 [>.............................] - ETA: 26:49 - loss: 0.9713 - ppl: 2.7895 - accu: 0.597 - ETA: 26:48 - loss: 0.9711 - ppl: 2.7887 - accu: 0.598 - ETA: 26:48 - loss: 0.9709 - ppl: 2.7880 - accu: 0.598 - ETA: 26:47 - loss: 0.9706 - ppl: 2.7872 - accu: 0.598 - ETA: 26:47 - loss: 0.9704 - ppl: 2.7864 - accu: 0.598 - ETA: 26:47 - loss: 0.9702 - ppl: 2.7858 - accu: 0.598 - ETA: 26:46 - loss: 0.9700 - ppl: 2.7850 - accu: 0.598 - ETA: 26:46 - loss: 0.9698 - ppl: 2.7843 - accu: 0.598 - ETA: 26:46 - loss: 0.9695 - ppl: 2.7836 - accu: 0.598 - ETA: 26:45 - loss: 0.9694 - ppl: 2.7829 - accu: 0.598 - ETA: 26:45 - loss: 0.9692 - ppl: 2.7823 - accu: 0.598 - ETA: 26:44 - loss: 0.9689 - ppl: 2.7815 - accu: 0.598 - ETA: 26:44 - loss: 0.9687 - ppl: 2.7807 - accu: 0.598 - ETA: 26:43 - loss: 0.9684 - ppl: 2.7799 - accu: 0.598 - ETA: 26:43 - loss: 0.9682 - ppl: 2.7791 - accu: 0.599 - ETA: 26:43 - loss: 0.9680 - ppl: 2.7785 - accu: 0.599 - ETA: 26:42 - loss: 0.9677 - ppl: 2.7776 - accu: 0.599 - ETA: 26:42 - loss: 0.9675 - ppl: 2.7769 - accu: 0.599 - ETA: 26:41 - loss: 0.9673 - ppl: 2.7762 - accu: 0.599 - ETA: 26:41 - loss: 0.9671 - ppl: 2.7756 - accu: 0.599 - ETA: 26:40 - loss: 0.9669 - ppl: 2.7748 - accu: 0.599 - ETA: 26:40 - loss: 0.9666 - ppl: 2.7740 - accu: 0.599 - ETA: 26:40 - loss: 0.9664 - ppl: 2.7732 - accu: 0.599 - ETA: 26:39 - loss: 0.9662 - ppl: 2.7725 - accu: 0.600 - ETA: 26:39 - loss: 0.9659 - ppl: 2.7718 - accu: 0.600 - ETA: 26:39 - loss: 0.9657 - ppl: 2.7710 - accu: 0.600 - ETA: 26:39 - loss: 0.9655 - ppl: 2.7703 - accu: 0.600 - ETA: 26:38 - loss: 0.9653 - ppl: 2.7695 - accu: 0.600 - ETA: 26:38 - loss: 0.9651 - ppl: 2.7690 - accu: 0.600 - ETA: 26:38 - loss: 0.9650 - ppl: 2.7685 - accu: 0.600 - ETA: 26:37 - loss: 0.9648 - ppl: 2.7679 - accu: 0.600 - ETA: 26:37 - loss: 0.9646 - ppl: 2.7672 - accu: 0.600 - ETA: 26:37 - loss: 0.9644 - ppl: 2.7664 - accu: 0.600 - ETA: 26:36 - loss: 0.9641 - ppl: 2.7657 - accu: 0.600 - ETA: 26:36 - loss: 0.9640 - ppl: 2.7651 - accu: 0.601 - ETA: 26:36 - loss: 0.9638 - ppl: 2.7645 - accu: 0.601 - ETA: 26:36 - loss: 0.9636 - ppl: 2.7639 - accu: 0.601 - ETA: 26:35 - loss: 0.9634 - ppl: 2.7632 - accu: 0.601 - ETA: 26:35 - loss: 0.9632 - ppl: 2.7625 - accu: 0.601 - ETA: 26:34 - loss: 0.9629 - ppl: 2.7617 - accu: 0.601 - ETA: 26:34 - loss: 0.9628 - ppl: 2.7611 - accu: 0.601 - ETA: 26:34 - loss: 0.9625 - ppl: 2.7603 - accu: 0.601 - ETA: 26:33 - loss: 0.9623 - ppl: 2.7595 - accu: 0.601 - ETA: 26:33 - loss: 0.9621 - ppl: 2.7588 - accu: 0.601 - ETA: 26:33 - loss: 0.9618 - ppl: 2.7581 - accu: 0.601 - ETA: 26:32 - loss: 0.9616 - ppl: 2.7574 - accu: 0.602 - ETA: 26:32 - loss: 0.9614 - ppl: 2.7567 - accu: 0.602 - ETA: 26:31 - loss: 0.9612 - ppl: 2.7561 - accu: 0.602 - ETA: 26:31 - loss: 0.9610 - ppl: 2.7554 - accu: 0.602 - ETA: 26:31 - loss: 0.9608 - ppl: 2.7547 - accu: 0.602 - ETA: 26:30 - loss: 0.9606 - ppl: 2.7541 - accu: 0.602 - ETA: 26:30 - loss: 0.9605 - ppl: 2.7535 - accu: 0.602 - ETA: 26:30 - loss: 0.9602 - ppl: 2.7528 - accu: 0.602 - ETA: 26:29 - loss: 0.9600 - ppl: 2.7520 - accu: 0.602 - ETA: 26:29 - loss: 0.9598 - ppl: 2.7514 - accu: 0.602 - ETA: 26:28 - loss: 0.9596 - ppl: 2.7508 - accu: 0.602 - ETA: 26:28 - loss: 0.9594 - ppl: 2.7500 - accu: 0.602 - ETA: 26:28 - loss: 0.9592 - ppl: 2.7494 - accu: 0.603 - ETA: 26:27 - loss: 0.9589 - ppl: 2.7486 - accu: 0.603 - ETA: 26:27 - loss: 0.9587 - ppl: 2.7479 - accu: 0.603 - ETA: 26:27 - loss: 0.9585 - ppl: 2.7472 - accu: 0.603 - ETA: 26:26 - loss: 0.9583 - ppl: 2.7466 - accu: 0.603 - ETA: 26:26 - loss: 0.9581 - ppl: 2.7459 - accu: 0.603 - ETA: 26:26 - loss: 0.9579 - ppl: 2.7452 - accu: 0.603 - ETA: 26:25 - loss: 0.9577 - ppl: 2.7445 - accu: 0.603 - ETA: 26:25 - loss: 0.9575 - ppl: 2.7439 - accu: 0.603 - ETA: 26:24 - loss: 0.9573 - ppl: 2.7432 - accu: 0.603 - ETA: 26:24 - loss: 0.9571 - ppl: 2.7426 - accu: 0.603 - ETA: 26:24 - loss: 0.9569 - ppl: 2.7419 - accu: 0.604 - ETA: 26:24 - loss: 0.9566 - ppl: 2.7411 - accu: 0.604 - ETA: 26:23 - loss: 0.9564 - ppl: 2.7405 - accu: 0.604 - ETA: 26:23 - loss: 0.9562 - ppl: 2.7399 - accu: 0.604 - ETA: 26:23 - loss: 0.9560 - ppl: 2.7392 - accu: 0.604 - ETA: 26:22 - loss: 0.9558 - ppl: 2.7386 - accu: 0.604 - ETA: 26:22 - loss: 0.9556 - ppl: 2.7379 - accu: 0.604 - ETA: 26:22 - loss: 0.9554 - ppl: 2.7372 - accu: 0.604 - ETA: 26:21 - loss: 0.9552 - ppl: 2.7366 - accu: 0.604 - ETA: 26:21 - loss: 0.9551 - ppl: 2.7360 - accu: 0.604 - ETA: 26:21 - loss: 0.9549 - ppl: 2.7354 - accu: 0.605 - ETA: 26:21 - loss: 0.9547 - ppl: 2.7347 - accu: 0.605 - ETA: 26:21 - loss: 0.9545 - ppl: 2.7342 - accu: 0.605 - ETA: 26:20 - loss: 0.9543 - ppl: 2.7334 - accu: 0.605 - ETA: 26:20 - loss: 0.9541 - ppl: 2.7329 - accu: 0.605 - ETA: 26:20 - loss: 0.9539 - ppl: 2.7322 - accu: 0.605 - ETA: 26:19 - loss: 0.9536 - ppl: 2.7314 - accu: 0.605 - ETA: 26:19 - loss: 0.9535 - ppl: 2.7309 - accu: 0.605 - ETA: 26:19 - loss: 0.9533 - ppl: 2.7302 - accu: 0.605 - ETA: 26:18 - loss: 0.9530 - ppl: 2.7295 - accu: 0.605 - ETA: 26:18 - loss: 0.9528 - ppl: 2.7289 - accu: 0.605 - ETA: 26:17 - loss: 0.9526 - ppl: 2.7281 - accu: 0.606 - ETA: 26:17 - loss: 0.9524 - ppl: 2.7275 - accu: 0.606 - ETA: 26:17 - loss: 0.9522 - ppl: 2.7269 - accu: 0.606 - ETA: 26:17 - loss: 0.9520 - ppl: 2.7263 - accu: 0.606 - ETA: 26:16 - loss: 0.9518 - ppl: 2.7257 - accu: 0.606 - ETA: 26:16 - loss: 0.9516 - ppl: 2.7249 - accu: 0.606 - ETA: 26:16 - loss: 0.9514 - ppl: 2.7243 - accu: 0.606 - ETA: 26:15 - loss: 0.9512 - ppl: 2.7237 - accu: 0.606 - ETA: 26:15 - loss: 0.9510 - ppl: 2.7231 - accu: 0.606 - ETA: 26:15 - loss: 0.9508 - ppl: 2.7223 - accu: 0.606 - ETA: 26:14 - loss: 0.9506 - ppl: 2.7217 - accu: 0.607 - ETA: 26:14 - loss: 0.9503 - ppl: 2.7209 - accu: 0.607 - ETA: 26:14 - loss: 0.9500 - ppl: 2.7202 - accu: 0.607 - ETA: 26:13 - loss: 0.9498 - ppl: 2.7195 - accu: 0.607 - ETA: 26:13 - loss: 0.9496 - ppl: 2.7188 - accu: 0.607 - ETA: 26:13 - loss: 0.9494 - ppl: 2.7182 - accu: 0.607 - ETA: 26:12 - loss: 0.9492 - ppl: 2.7175 - accu: 0.607 - ETA: 26:12 - loss: 0.9490 - ppl: 2.7168 - accu: 0.607 - ETA: 26:12 - loss: 0.9488 - ppl: 2.7163 - accu: 0.607 - ETA: 26:12 - loss: 0.9486 - ppl: 2.7156 - accu: 0.607 - ETA: 26:12 - loss: 0.9484 - ppl: 2.7149 - accu: 0.608 - ETA: 26:11 - loss: 0.9482 - ppl: 2.7144 - accu: 0.608 - ETA: 26:11 - loss: 0.9480 - ppl: 2.7138 - accu: 0.608 - ETA: 26:11 - loss: 0.9479 - ppl: 2.7132 - accu: 0.608 - ETA: 26:10 - loss: 0.9477 - ppl: 2.7126 - accu: 0.608 - ETA: 26:10 - loss: 0.9475 - ppl: 2.7120 - accu: 0.608 - ETA: 26:10 - loss: 0.9473 - ppl: 2.7114 - accu: 0.608 - ETA: 26:09 - loss: 0.9471 - ppl: 2.7108 - accu: 0.608 - ETA: 26:09 - loss: 0.9469 - ppl: 2.7101 - accu: 0.608 - ETA: 26:08 - loss: 0.9466 - ppl: 2.7094 - accu: 0.608 - ETA: 26:08 - loss: 0.9465 - ppl: 2.7088 - accu: 0.608 - ETA: 26:08 - loss: 0.9463 - ppl: 2.7082 - accu: 0.609 - ETA: 26:07 - loss: 0.9461 - ppl: 2.7076 - accu: 0.609 - ETA: 26:07 - loss: 0.9459 - ppl: 2.7069 - accu: 0.609 - ETA: 26:07 - loss: 0.9457 - ppl: 2.7063 - accu: 0.609 - ETA: 26:06 - loss: 0.9455 - ppl: 2.7057 - accu: 0.609 - ETA: 26:06 - loss: 0.9453 - ppl: 2.7050 - accu: 0.609 - ETA: 26:06 - loss: 0.9450 - ppl: 2.7043 - accu: 0.609 - ETA: 26:05 - loss: 0.9449 - ppl: 2.7038 - accu: 0.609 - ETA: 26:05 - loss: 0.9446 - ppl: 2.7031 - accu: 0.609 - ETA: 26:05 - loss: 0.9445 - ppl: 2.7025 - accu: 0.609 - ETA: 26:05 - loss: 0.9442 - ppl: 2.7019 - accu: 0.609 - ETA: 26:04 - loss: 0.9440 - ppl: 2.7011 - accu: 0.609 - ETA: 26:04 - loss: 0.9438 - ppl: 2.7004 - accu: 0.610 - ETA: 26:04 - loss: 0.9436 - ppl: 2.6998 - accu: 0.610 - ETA: 26:03 - loss: 0.9434 - ppl: 2.6992 - accu: 0.610 - ETA: 26:03 - loss: 0.9432 - ppl: 2.6987 - accu: 0.610 - ETA: 26:03 - loss: 0.9431 - ppl: 2.6982 - accu: 0.610 - ETA: 26:02 - loss: 0.9428 - ppl: 2.6975 - accu: 0.610 - ETA: 26:02 - loss: 0.9427 - ppl: 2.6970 - accu: 0.610 - ETA: 26:02 - loss: 0.9425 - ppl: 2.6964 - accu: 0.610 - ETA: 26:01 - loss: 0.9423 - ppl: 2.6958 - accu: 0.610 - ETA: 26:01 - loss: 0.9422 - ppl: 2.6953 - accu: 0.610 - ETA: 26:01 - loss: 0.9419 - ppl: 2.6946 - accu: 0.610 - ETA: 26:01 - loss: 0.9416 - ppl: 2.6938 - accu: 0.610 - ETA: 26:00 - loss: 0.9415 - ppl: 2.6932 - accu: 0.611 - ETA: 26:00 - loss: 0.9413 - ppl: 2.6927 - accu: 0.6111"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 16152/400000 [>.............................] - ETA: 26:00 - loss: 0.9411 - ppl: 2.6920 - accu: 0.611 - ETA: 26:00 - loss: 0.9409 - ppl: 2.6913 - accu: 0.611 - ETA: 26:00 - loss: 0.9407 - ppl: 2.6908 - accu: 0.611 - ETA: 26:00 - loss: 0.9406 - ppl: 2.6904 - accu: 0.611 - ETA: 26:00 - loss: 0.9404 - ppl: 2.6898 - accu: 0.611 - ETA: 25:59 - loss: 0.9401 - ppl: 2.6891 - accu: 0.611 - ETA: 25:59 - loss: 0.9400 - ppl: 2.6885 - accu: 0.611 - ETA: 25:59 - loss: 0.9397 - ppl: 2.6878 - accu: 0.611 - ETA: 25:59 - loss: 0.9395 - ppl: 2.6871 - accu: 0.611 - ETA: 25:59 - loss: 0.9393 - ppl: 2.6865 - accu: 0.612 - ETA: 25:58 - loss: 0.9391 - ppl: 2.6859 - accu: 0.612 - ETA: 25:58 - loss: 0.9389 - ppl: 2.6853 - accu: 0.612 - ETA: 25:58 - loss: 0.9387 - ppl: 2.6846 - accu: 0.612 - ETA: 25:58 - loss: 0.9385 - ppl: 2.6841 - accu: 0.612 - ETA: 25:58 - loss: 0.9383 - ppl: 2.6834 - accu: 0.612 - ETA: 25:58 - loss: 0.9381 - ppl: 2.6828 - accu: 0.612 - ETA: 25:57 - loss: 0.9378 - ppl: 2.6821 - accu: 0.612 - ETA: 25:57 - loss: 0.9376 - ppl: 2.6814 - accu: 0.612 - ETA: 25:57 - loss: 0.9373 - ppl: 2.6807 - accu: 0.612 - ETA: 25:57 - loss: 0.9373 - ppl: 2.6804 - accu: 0.612 - ETA: 25:56 - loss: 0.9371 - ppl: 2.6798 - accu: 0.613 - ETA: 25:56 - loss: 0.9368 - ppl: 2.6791 - accu: 0.613 - ETA: 25:56 - loss: 0.9366 - ppl: 2.6784 - accu: 0.613 - ETA: 25:56 - loss: 0.9364 - ppl: 2.6778 - accu: 0.613 - ETA: 25:55 - loss: 0.9362 - ppl: 2.6772 - accu: 0.613 - ETA: 25:55 - loss: 0.9361 - ppl: 2.6767 - accu: 0.613 - ETA: 25:55 - loss: 0.9359 - ppl: 2.6761 - accu: 0.613 - ETA: 25:54 - loss: 0.9356 - ppl: 2.6755 - accu: 0.613 - ETA: 25:54 - loss: 0.9354 - ppl: 2.6748 - accu: 0.613 - ETA: 25:54 - loss: 0.9352 - ppl: 2.6743 - accu: 0.613 - ETA: 25:53 - loss: 0.9350 - ppl: 2.6736 - accu: 0.614 - ETA: 25:53 - loss: 0.9348 - ppl: 2.6730 - accu: 0.614 - ETA: 25:53 - loss: 0.9346 - ppl: 2.6723 - accu: 0.614 - ETA: 25:53 - loss: 0.9344 - ppl: 2.6717 - accu: 0.614 - ETA: 25:53 - loss: 0.9342 - ppl: 2.6711 - accu: 0.614 - ETA: 25:53 - loss: 0.9340 - ppl: 2.6705 - accu: 0.614 - ETA: 25:53 - loss: 0.9337 - ppl: 2.6699 - accu: 0.614 - ETA: 25:52 - loss: 0.9336 - ppl: 2.6695 - accu: 0.614 - ETA: 25:52 - loss: 0.9334 - ppl: 2.6689 - accu: 0.614 - ETA: 25:52 - loss: 0.9332 - ppl: 2.6683 - accu: 0.614 - ETA: 25:52 - loss: 0.9330 - ppl: 2.6677 - accu: 0.614 - ETA: 25:52 - loss: 0.9328 - ppl: 2.6671 - accu: 0.614 - ETA: 25:52 - loss: 0.9326 - ppl: 2.6665 - accu: 0.615 - ETA: 25:52 - loss: 0.9325 - ppl: 2.6659 - accu: 0.615 - ETA: 25:51 - loss: 0.9323 - ppl: 2.6654 - accu: 0.615 - ETA: 25:51 - loss: 0.9321 - ppl: 2.6648 - accu: 0.615 - ETA: 25:51 - loss: 0.9319 - ppl: 2.6642 - accu: 0.615 - ETA: 25:51 - loss: 0.9317 - ppl: 2.6637 - accu: 0.615 - ETA: 25:51 - loss: 0.9316 - ppl: 2.6631 - accu: 0.615 - ETA: 25:51 - loss: 0.9313 - ppl: 2.6625 - accu: 0.615 - ETA: 25:51 - loss: 0.9311 - ppl: 2.6618 - accu: 0.615 - ETA: 25:51 - loss: 0.9309 - ppl: 2.6612 - accu: 0.615 - ETA: 25:51 - loss: 0.9307 - ppl: 2.6606 - accu: 0.615 - ETA: 25:51 - loss: 0.9304 - ppl: 2.6599 - accu: 0.616 - ETA: 25:50 - loss: 0.9302 - ppl: 2.6592 - accu: 0.616 - ETA: 25:50 - loss: 0.9300 - ppl: 2.6587 - accu: 0.616 - ETA: 25:50 - loss: 0.9299 - ppl: 2.6582 - accu: 0.616 - ETA: 25:49 - loss: 0.9296 - ppl: 2.6576 - accu: 0.616 - ETA: 25:49 - loss: 0.9295 - ppl: 2.6570 - accu: 0.616 - ETA: 25:49 - loss: 0.9293 - ppl: 2.6564 - accu: 0.616 - ETA: 25:49 - loss: 0.9290 - ppl: 2.6557 - accu: 0.616 - ETA: 25:48 - loss: 0.9288 - ppl: 2.6550 - accu: 0.616 - ETA: 25:48 - loss: 0.9286 - ppl: 2.6544 - accu: 0.616 - ETA: 25:48 - loss: 0.9284 - ppl: 2.6539 - accu: 0.616 - ETA: 25:47 - loss: 0.9282 - ppl: 2.6534 - accu: 0.616 - ETA: 25:47 - loss: 0.9281 - ppl: 2.6528 - accu: 0.617 - ETA: 25:47 - loss: 0.9279 - ppl: 2.6523 - accu: 0.617 - ETA: 25:47 - loss: 0.9277 - ppl: 2.6518 - accu: 0.617 - ETA: 25:46 - loss: 0.9276 - ppl: 2.6513 - accu: 0.617 - ETA: 25:46 - loss: 0.9274 - ppl: 2.6508 - accu: 0.617 - ETA: 25:46 - loss: 0.9272 - ppl: 2.6502 - accu: 0.617 - ETA: 25:46 - loss: 0.9271 - ppl: 2.6498 - accu: 0.617 - ETA: 25:46 - loss: 0.9269 - ppl: 2.6492 - accu: 0.617 - ETA: 25:46 - loss: 0.9266 - ppl: 2.6485 - accu: 0.617 - ETA: 25:46 - loss: 0.9265 - ppl: 2.6482 - accu: 0.617 - ETA: 25:46 - loss: 0.9263 - ppl: 2.6475 - accu: 0.617 - ETA: 25:45 - loss: 0.9262 - ppl: 2.6470 - accu: 0.617 - ETA: 25:45 - loss: 0.9260 - ppl: 2.6465 - accu: 0.618 - ETA: 25:45 - loss: 0.9257 - ppl: 2.6458 - accu: 0.618 - ETA: 25:45 - loss: 0.9255 - ppl: 2.6452 - accu: 0.618 - ETA: 25:44 - loss: 0.9253 - ppl: 2.6446 - accu: 0.618 - ETA: 25:44 - loss: 0.9251 - ppl: 2.6439 - accu: 0.618 - ETA: 25:44 - loss: 0.9248 - ppl: 2.6432 - accu: 0.618 - ETA: 25:43 - loss: 0.9245 - ppl: 2.6424 - accu: 0.618 - ETA: 25:43 - loss: 0.9243 - ppl: 2.6417 - accu: 0.618 - ETA: 25:43 - loss: 0.9241 - ppl: 2.6411 - accu: 0.618 - ETA: 25:43 - loss: 0.9238 - ppl: 2.6405 - accu: 0.619 - ETA: 25:43 - loss: 0.9236 - ppl: 2.6399 - accu: 0.619 - ETA: 25:43 - loss: 0.9234 - ppl: 2.6393 - accu: 0.619 - ETA: 25:43 - loss: 0.9232 - ppl: 2.6388 - accu: 0.619 - ETA: 25:43 - loss: 0.9231 - ppl: 2.6382 - accu: 0.619 - ETA: 25:42 - loss: 0.9230 - ppl: 2.6380 - accu: 0.619 - ETA: 25:42 - loss: 0.9228 - ppl: 2.6373 - accu: 0.619 - ETA: 25:42 - loss: 0.9225 - ppl: 2.6366 - accu: 0.619 - ETA: 25:41 - loss: 0.9223 - ppl: 2.6361 - accu: 0.619 - ETA: 25:41 - loss: 0.9222 - ppl: 2.6356 - accu: 0.619 - ETA: 25:41 - loss: 0.9220 - ppl: 2.6352 - accu: 0.619 - ETA: 25:41 - loss: 0.9218 - ppl: 2.6345 - accu: 0.619 - ETA: 25:40 - loss: 0.9216 - ppl: 2.6340 - accu: 0.619 - ETA: 25:40 - loss: 0.9215 - ppl: 2.6336 - accu: 0.620 - ETA: 25:40 - loss: 0.9213 - ppl: 2.6330 - accu: 0.620 - ETA: 25:40 - loss: 0.9212 - ppl: 2.6325 - accu: 0.620 - ETA: 25:39 - loss: 0.9209 - ppl: 2.6318 - accu: 0.620 - ETA: 25:39 - loss: 0.9207 - ppl: 2.6313 - accu: 0.620 - ETA: 25:39 - loss: 0.9205 - ppl: 2.6306 - accu: 0.620 - ETA: 25:39 - loss: 0.9204 - ppl: 2.6302 - accu: 0.620 - ETA: 25:39 - loss: 0.9202 - ppl: 2.6296 - accu: 0.620 - ETA: 25:38 - loss: 0.9200 - ppl: 2.6290 - accu: 0.620 - ETA: 25:38 - loss: 0.9197 - ppl: 2.6284 - accu: 0.620 - ETA: 25:38 - loss: 0.9195 - ppl: 2.6279 - accu: 0.620 - ETA: 25:38 - loss: 0.9193 - ppl: 2.6273 - accu: 0.620 - ETA: 25:38 - loss: 0.9191 - ppl: 2.6267 - accu: 0.621 - ETA: 25:38 - loss: 0.9189 - ppl: 2.6261 - accu: 0.621 - ETA: 25:37 - loss: 0.9187 - ppl: 2.6255 - accu: 0.621 - ETA: 25:37 - loss: 0.9185 - ppl: 2.6249 - accu: 0.621 - ETA: 25:37 - loss: 0.9183 - ppl: 2.6243 - accu: 0.621 - ETA: 25:37 - loss: 0.9182 - ppl: 2.6239 - accu: 0.621 - ETA: 25:36 - loss: 0.9180 - ppl: 2.6234 - accu: 0.621 - ETA: 25:36 - loss: 0.9177 - ppl: 2.6227 - accu: 0.621 - ETA: 25:36 - loss: 0.9176 - ppl: 2.6223 - accu: 0.621 - ETA: 25:36 - loss: 0.9174 - ppl: 2.6217 - accu: 0.621 - ETA: 25:36 - loss: 0.9172 - ppl: 2.6211 - accu: 0.621 - ETA: 25:36 - loss: 0.9169 - ppl: 2.6204 - accu: 0.622 - ETA: 25:36 - loss: 0.9167 - ppl: 2.6199 - accu: 0.622 - ETA: 25:36 - loss: 0.9166 - ppl: 2.6193 - accu: 0.622 - ETA: 25:36 - loss: 0.9164 - ppl: 2.6188 - accu: 0.622 - ETA: 25:36 - loss: 0.9162 - ppl: 2.6184 - accu: 0.622 - ETA: 25:36 - loss: 0.9160 - ppl: 2.6178 - accu: 0.622 - ETA: 25:36 - loss: 0.9158 - ppl: 2.6173 - accu: 0.622 - ETA: 25:36 - loss: 0.9158 - ppl: 2.6170 - accu: 0.622 - ETA: 25:36 - loss: 0.9155 - ppl: 2.6163 - accu: 0.622 - ETA: 25:35 - loss: 0.9154 - ppl: 2.6159 - accu: 0.622 - ETA: 25:35 - loss: 0.9152 - ppl: 2.6154 - accu: 0.622 - ETA: 25:35 - loss: 0.9150 - ppl: 2.6148 - accu: 0.622 - ETA: 25:34 - loss: 0.9148 - ppl: 2.6142 - accu: 0.622 - ETA: 25:34 - loss: 0.9146 - ppl: 2.6136 - accu: 0.623 - ETA: 25:34 - loss: 0.9144 - ppl: 2.6131 - accu: 0.6231"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-26-ec2a2f0ec2b8>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#batch_size=64, epochs=30, \\\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0ms2s\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0minput_data\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutput_data\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m8\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalidation_split\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\envs\\py36\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[0;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1039\u001b[1;33m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[0;32m   1040\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\py36\\lib\\site-packages\\keras\\engine\\training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[1;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[0;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    198\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 199\u001b[1;33m                 \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    201\u001b[0m                 \u001b[1;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mo\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\py36\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2713\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2714\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2715\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2716\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2717\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\py36\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2674\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2675\u001b[1;33m             \u001b[0mfetched\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2676\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2677\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1437\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[0;32m   1438\u001b[0m               \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1439\u001b[1;33m               run_metadata_ptr)\n\u001b[0m\u001b[0;32m   1440\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1441\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#batch_size=64, epochs=30, \\\n",
    "s2s.model.fit([input_data, output_data], None, batch_size=8, epochs=1, validation_split=0.2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py36",
   "language": "python",
   "name": "py36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}