10_rnn_language_modeling.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "10_rnn_language_modeling.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/iamdsc/deep_learning/blob/master/10_rnn_language_modeling.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "zLdjPD9PvPKV",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Implementing RNN for character-level language modeling in Tensorflow"
      ]
    },
    {
      "metadata": {
        "id": "MhZEeGepvIHI",
        "colab_type": "code",
        "outputId": "7d848ffb-2c5f-43d2-90bc-e6c250b2ec39",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 219
        }
      },
      "cell_type": "code",
      "source": [
        "# downloading the text data\n",
        "# Data: The Tragedie of Hamlet by William Shakespeare in plain text format\n",
        "!wget https://github.com/iamdsc/deep_learning/blob/master/data/pg2265.txt"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-02-28 16:42:57--  https://github.com/iamdsc/deep_learning/blob/master/data/pg2265.txt\n",
            "Resolving github.com (github.com)... 192.30.253.112, 192.30.253.113\n",
            "Connecting to github.com (github.com)|192.30.253.112|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: unspecified [text/html]\n",
            "Saving to: ‘pg2265.txt.1’\n",
            "\n",
            "pg2265.txt.1            [  <=>               ]   1.15M  2.61MB/s    in 0.4s    \n",
            "\n",
            "2019-02-28 16:42:58 (2.61 MB/s) - ‘pg2265.txt.1’ saved [1207044]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "KEj80YdtwiUv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "# Reading and processing the data\n",
        "with open('pg2265.txt','r',encoding='utf-8') as f:\n",
        "  text=f.read()\n",
        "\n",
        "text=text[15858:] # removes beginning portion of text\n",
        "chars=set(text)   # to get unique characters\n",
        "char2int={ch:i for i,ch in enumerate(chars)} # dict mapping each char to int\n",
        "int2char=dict(enumerate(chars)) # reverse mapping\n",
        "text_ints=np.array([char2int[ch] for ch in text], dtype=np.int32)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "HKC4cjKZ6VqS",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def reshape_data(sequence, batch_size, num_steps):\n",
        "  tot_batch_length=batch_size*num_steps\n",
        "  num_batches=int(len(sequence)/tot_batch_length)\n",
        "  if num_batches*tot_batch_length + 1 > len(sequence):\n",
        "    num_batches=num_batches-1\n",
        "  \n",
        "  # Truncate sequence at the end to get rid of remaining chars\n",
        "  # that don't make a full batch\n",
        "  x=sequence[0:num_batches*tot_batch_length]\n",
        "  y=sequence[1:num_batches*tot_batch_length+1]\n",
        "  \n",
        "  # split x and y into a list of batches of sequences\n",
        "  x_batch_splits=np.split(x, batch_size)\n",
        "  y_batch_splits=np.split(y, batch_size)\n",
        "  \n",
        "  # stack the batches together\n",
        "  x=np.stack(x_batch_splits)\n",
        "  y=np.stack(y_batch_splits)\n",
        "  \n",
        "  return x, y"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8eW1041XNbgp",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def create_batch_generator(data_x, data_y, num_steps):\n",
        "  batch_size, tot_batch_length=data_x.shape\n",
        "  num_batches=int(tot_batch_length/num_steps)\n",
        "  for b in range(num_batches):\n",
        "    yield (data_x[:,b*num_steps:(b+1)*num_steps],data_y[:,b*num_steps:(b+1)*num_steps])    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ilgT7k47UKrg",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Building the character-level RNN Model"
      ]
    },
    {
      "metadata": {
        "id": "EQ6q8blNPRjc",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "import os\n",
        "\n",
        "class CharRNN(object):\n",
        "  def __init__(self,num_classes,batch_size=64,num_steps=100,\n",
        "               lstm_size=128,num_layers=1,learning_rate=0.001,\n",
        "               keep_prob=0.5,grad_clip=5,sampling=False):\n",
        "    self.num_classes=num_classes\n",
        "    self.batch_size=batch_size\n",
        "    self.num_steps=num_steps\n",
        "    self.lstm_size=lstm_size\n",
        "    self.num_layers=num_layers\n",
        "    self.learning_rate=learning_rate\n",
        "    self.keep_prob=keep_prob\n",
        "    self.grad_clip=grad_clip\n",
        "    \n",
        "    self.g=tf.Graph()\n",
        "    with self.g.as_default():\n",
        "      tf.set_random_seed(123)\n",
        "      self.build(sampling=sampling)\n",
        "      self.saver=tf.train.Saver()\n",
        "      self.init_op=tf.global_variables_initializer()\n",
        "  \n",
        "  def build(self,sampling):\n",
        "    if sampling==True:\n",
        "      batch_size, num_steps=1, 1\n",
        "    else:\n",
        "      batch_size=self.batch_size\n",
        "      num_steps=self.num_steps\n",
        "    \n",
        "    tf_x=tf.placeholder(tf.int32,shape=[batch_size,num_steps],name='tf_x')\n",
        "    tf_y=tf.placeholder(tf.int32,shape=[batch_size,num_steps],name='tf_y')\n",
        "    tf_keepprob=tf.placeholder(tf.float32,name='tf_keepprob')\n",
        "    \n",
        "    # One-hot encoding\n",
        "    x_onehot=tf.one_hot(tf_x,depth=self.num_classes)\n",
        "    y_onehot=tf.one_hot(tf_y,depth=self.num_classes)\n",
        "    \n",
        "    # Build multilayer RNN cells\n",
        "    cells=tf.contrib.rnn.MultiRNNCell([\n",
        "          tf.contrib.rnn.DropoutWrapper(\n",
        "          tf.contrib.rnn.BasicLSTMCell(self.lstm_size),output_keep_prob=tf_keepprob)\n",
        "          for _ in range(self.num_layers)])\n",
        "    \n",
        "    # Define the initial state\n",
        "    self.initial_state=cells.zero_state(batch_size,tf.float32)\n",
        "    \n",
        "    # Run each sequence step through the rnn\n",
        "    lstm_outputs,self.final_state=tf.nn.dynamic_rnn(cells,x_onehot,initial_state=self.initial_state)\n",
        "    \n",
        "    print(' << lstm_outputs >> ',lstm_outputs)\n",
        "    \n",
        "    seq_output_reshaped=tf.reshape(lstm_outputs,shape=[-1,self.lstm_size],name='seq_output_reshaped')\n",
        "    \n",
        "    logits=tf.layers.dense(inputs=seq_output_reshaped,units=self.num_classes,activation=None,name='logits')\n",
        "    \n",
        "    probs=tf.nn.softmax(logits,name='probabilities')\n",
        "    \n",
        "    y_reshaped=tf.reshape(y_onehot,shape=[-1,self.num_classes],name='y_reshaped')\n",
        "    \n",
        "    cost=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits,labels=y_reshaped),name='cost')\n",
        "    \n",
        "    # Gradient clipping to avoid 'exploding gradients'\n",
        "    tvars=tf.trainable_variables()\n",
        "    grads,_=tf.clip_by_global_norm(tf.gradients(cost,tvars),self.grad_clip)\n",
        "    \n",
        "    optimizer=tf.train.AdamOptimizer(self.learning_rate)\n",
        "    train_op=optimizer.apply_gradients(zip(grads,tvars),name='train_op')\n",
        "  \n",
        "  def train(self,train_x,train_y,num_epochs,ckpt_dir='./model/'):\n",
        "    # Create the checkpoint directory if it doesn't exist\n",
        "    if not os.path.exists(ckpt_dir):\n",
        "      os.mkdir(ckpt_dir)\n",
        "    \n",
        "    with tf.Session(graph=self.g) as sess:\n",
        "      sess.run(self.init_op)\n",
        "      \n",
        "      n_batches=int(train_x.shape[1]/self.num_steps)\n",
        "      iterations=n_batches*num_epochs\n",
        "      for epoch in range(num_epochs):\n",
        "        # Train network\n",
        "        new_state=sess.run(self.initial_state)\n",
        "        loss=0\n",
        "        ## Mini-batch generator\n",
        "        bgen=create_batch_generator(train_x,train_y,self.num_steps)\n",
        "        for b,(batch_x,batch_y) in enumerate(bgen,1):\n",
        "          iteration=epoch*n_batches + b\n",
        "          \n",
        "          feed={'tf_x:0':batch_x, 'tf_y:0':batch_y, 'tf_keepprob:0':self.keep_prob, self.initial_state:new_state}\n",
        "          batch_cost,_,new_state=sess.run(['cost:0','train_op',self.final_state],feed_dict=feed)\n",
        "          \n",
        "          if iteration % 10==0:\n",
        "            print('Epoch %d/%d Iteration %d | Training loss: %.4f' % (epoch+1,num_epochs,iteration,batch_cost))\n",
        "          \n",
        "      ## Save the trained model\n",
        "      self.saver.save(sess,os.path.join(ckpt_dir,'language_modeling.ckpt'))\n",
        "  \n",
        "  def sample(self,output_length,ckpt_dir,starter_seq='The '):\n",
        "    observed_seq=[ch for ch in starter_seq]\n",
        "    with tf.Session(graph=self.g) as sess:\n",
        "      self.saver.restore(sess,tf.train.latest_checkpoint(ckpt_dir))\n",
        "      # run the model using the starter sequence\n",
        "      new_state=sess.run(self.initial_state)\n",
        "      for ch in starter_seq:\n",
        "        x=np.zeros((1,1))\n",
        "        x[0,0]=char2int[ch]\n",
        "        \n",
        "        feed={'tf_x:0':x,'tf_keepprob:0':1.0,self.initial_state:new_state}\n",
        "        proba,new_state=sess.run(['probabilities:0',self.final_state],feed_dict=feed)\n",
        "        \n",
        "      ch_id=get_top_char(proba,len(chars))\n",
        "      observed_seq.append(int2char[ch_id])\n",
        "      \n",
        "      # run the model using the updated observed_seq\n",
        "      for i in range(output_length):\n",
        "        x[0, 0]=ch_id\n",
        "        feed={'tf_x:0':x,'tf_keepprob:0':1.0,self.initial_state:new_state}\n",
        "        proba, new_state=sess.run(['probabilities:0',self.final_state],feed_dict=feed)\n",
        "        \n",
        "        ch_id=get_top_char(proba,len(chars))\n",
        "        observed_seq.append(int2char[ch_id])\n",
        "    \n",
        "    return ''.join(observed_seq)\n",
        "\n",
        "def get_top_char(probas,char_size,top_n=5):\n",
        "  p=np.squeeze(probas)\n",
        "  p[np.argsort(p)[:-top_n]]=0.0\n",
        "  p=p/np.sum(p)\n",
        "  ch_id=np.random.choice(char_size,1,p=p)[0]\n",
        "  return ch_id\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "vkYKVe5DqCA3",
        "colab_type": "code",
        "outputId": "8d9ab7dc-5e59-4b06-84a7-baa0790a26fa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34823
        }
      },
      "cell_type": "code",
      "source": [
        "# Train the model\n",
        "batch_size=64\n",
        "num_steps=100\n",
        "train_x, train_y = reshape_data(text_ints, batch_size, num_steps)\n",
        "\n",
        "rnn=CharRNN(num_classes=len(chars),batch_size=batch_size)\n",
        "rnn.train(train_x,train_y,num_epochs=100,ckpt_dir='./model-100/')\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n",
            "WARNING:tensorflow:From <ipython-input-9-3c1dc0d6e4a8>:43: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n",
            "WARNING:tensorflow:From <ipython-input-9-3c1dc0d6e4a8>:43: MultiRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n",
            "WARNING:tensorflow:From <ipython-input-9-3c1dc0d6e4a8>:49: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/tensor_array_ops.py:162: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Colocations handled automatically by placer.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn_cell_impl.py:1259: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
            " << lstm_outputs >>  Tensor(\"rnn/transpose_1:0\", shape=(64, 100, 128), dtype=float32)\n",
            "WARNING:tensorflow:From <ipython-input-9-3c1dc0d6e4a8>:55: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.dense instead.\n",
            "WARNING:tensorflow:From <ipython-input-9-3c1dc0d6e4a8>:61: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "\n",
            "Future major versions of TensorFlow will allow gradients to flow\n",
            "into the labels input on backprop by default.\n",
            "\n",
            "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
            "\n",
            "Epoch 1/100 Iteration 10 | Training loss: 3.9934\n",
            "Epoch 1/100 Iteration 20 | Training loss: 3.5513\n",
            "Epoch 1/100 Iteration 30 | Training loss: 3.4184\n",
            "Epoch 1/100 Iteration 40 | Training loss: 3.3085\n",
            "Epoch 1/100 Iteration 50 | Training loss: 3.2187\n",
            "Epoch 1/100 Iteration 60 | Training loss: 3.0831\n",
            "Epoch 1/100 Iteration 70 | Training loss: 3.0047\n",
            "Epoch 1/100 Iteration 80 | Training loss: 2.9128\n",
            "Epoch 1/100 Iteration 90 | Training loss: 2.7710\n",
            "Epoch 1/100 Iteration 100 | Training loss: 2.7055\n",
            "Epoch 1/100 Iteration 110 | Training loss: 2.6414\n",
            "Epoch 1/100 Iteration 120 | Training loss: 2.5759\n",
            "Epoch 1/100 Iteration 130 | Training loss: 2.4578\n",
            "Epoch 1/100 Iteration 140 | Training loss: 2.3807\n",
            "Epoch 1/100 Iteration 150 | Training loss: 2.3616\n",
            "Epoch 1/100 Iteration 160 | Training loss: 2.2886\n",
            "Epoch 1/100 Iteration 170 | Training loss: 2.3439\n",
            "Epoch 1/100 Iteration 180 | Training loss: 2.0438\n",
            "Epoch 2/100 Iteration 190 | Training loss: 1.9999\n",
            "Epoch 2/100 Iteration 200 | Training loss: 1.9877\n",
            "Epoch 2/100 Iteration 210 | Training loss: 1.9842\n",
            "Epoch 2/100 Iteration 220 | Training loss: 1.8872\n",
            "Epoch 2/100 Iteration 230 | Training loss: 1.9047\n",
            "Epoch 2/100 Iteration 240 | Training loss: 1.7319\n",
            "Epoch 2/100 Iteration 250 | Training loss: 1.6574\n",
            "Epoch 2/100 Iteration 260 | Training loss: 1.6295\n",
            "Epoch 2/100 Iteration 270 | Training loss: 1.6321\n",
            "Epoch 2/100 Iteration 280 | Training loss: 1.5785\n",
            "Epoch 2/100 Iteration 290 | Training loss: 1.3769\n",
            "Epoch 2/100 Iteration 300 | Training loss: 1.4532\n",
            "Epoch 2/100 Iteration 310 | Training loss: 1.5076\n",
            "Epoch 2/100 Iteration 320 | Training loss: 1.3318\n",
            "Epoch 2/100 Iteration 330 | Training loss: 1.3768\n",
            "Epoch 2/100 Iteration 340 | Training loss: 1.2634\n",
            "Epoch 2/100 Iteration 350 | Training loss: 1.2965\n",
            "Epoch 2/100 Iteration 360 | Training loss: 1.3027\n",
            "Epoch 2/100 Iteration 370 | Training loss: 1.1140\n",
            "Epoch 3/100 Iteration 380 | Training loss: 1.3070\n",
            "Epoch 3/100 Iteration 390 | Training loss: 0.9926\n",
            "Epoch 3/100 Iteration 400 | Training loss: 1.1742\n",
            "Epoch 3/100 Iteration 410 | Training loss: 1.0961\n",
            "Epoch 3/100 Iteration 420 | Training loss: 1.1020\n",
            "Epoch 3/100 Iteration 430 | Training loss: 1.0552\n",
            "Epoch 3/100 Iteration 440 | Training loss: 1.1203\n",
            "Epoch 3/100 Iteration 450 | Training loss: 0.9886\n",
            "Epoch 3/100 Iteration 460 | Training loss: 0.8766\n",
            "Epoch 3/100 Iteration 470 | Training loss: 0.8721\n",
            "Epoch 3/100 Iteration 480 | Training loss: 0.9658\n",
            "Epoch 3/100 Iteration 490 | Training loss: 0.9286\n",
            "Epoch 3/100 Iteration 500 | Training loss: 0.9339\n",
            "Epoch 3/100 Iteration 510 | Training loss: 0.8651\n",
            "Epoch 3/100 Iteration 520 | Training loss: 0.9660\n",
            "Epoch 3/100 Iteration 530 | Training loss: 0.8600\n",
            "Epoch 3/100 Iteration 540 | Training loss: 1.0490\n",
            "Epoch 3/100 Iteration 550 | Training loss: 0.7873\n",
            "Epoch 4/100 Iteration 560 | Training loss: 0.8028\n",
            "Epoch 4/100 Iteration 570 | Training loss: 0.9162\n",
            "Epoch 4/100 Iteration 580 | Training loss: 0.9289\n",
            "Epoch 4/100 Iteration 590 | Training loss: 0.8501\n",
            "Epoch 4/100 Iteration 600 | Training loss: 0.8783\n",
            "Epoch 4/100 Iteration 610 | Training loss: 0.7954\n",
            "Epoch 4/100 Iteration 620 | Training loss: 0.8640\n",
            "Epoch 4/100 Iteration 630 | Training loss: 0.7541\n",
            "Epoch 4/100 Iteration 640 | Training loss: 0.7574\n",
            "Epoch 4/100 Iteration 650 | Training loss: 0.8109\n",
            "Epoch 4/100 Iteration 660 | Training loss: 0.6981\n",
            "Epoch 4/100 Iteration 670 | Training loss: 0.7268\n",
            "Epoch 4/100 Iteration 680 | Training loss: 0.7700\n",
            "Epoch 4/100 Iteration 690 | Training loss: 0.7443\n",
            "Epoch 4/100 Iteration 700 | Training loss: 0.7767\n",
            "Epoch 4/100 Iteration 710 | Training loss: 0.6832\n",
            "Epoch 4/100 Iteration 720 | Training loss: 0.7771\n",
            "Epoch 4/100 Iteration 730 | Training loss: 0.7541\n",
            "Epoch 4/100 Iteration 740 | Training loss: 0.7445\n",
            "Epoch 5/100 Iteration 750 | Training loss: 0.7817\n",
            "Epoch 5/100 Iteration 760 | Training loss: 0.7465\n",
            "Epoch 5/100 Iteration 770 | Training loss: 0.8010\n",
            "Epoch 5/100 Iteration 780 | Training loss: 0.6978\n",
            "Epoch 5/100 Iteration 790 | Training loss: 0.7457\n",
            "Epoch 5/100 Iteration 800 | Training loss: 0.7636\n",
            "Epoch 5/100 Iteration 810 | Training loss: 0.7799\n",
            "Epoch 5/100 Iteration 820 | Training loss: 0.6471\n",
            "Epoch 5/100 Iteration 830 | Training loss: 0.6633\n",
            "Epoch 5/100 Iteration 840 | Training loss: 0.6675\n",
            "Epoch 5/100 Iteration 850 | Training loss: 0.7388\n",
            "Epoch 5/100 Iteration 860 | Training loss: 0.7824\n",
            "Epoch 5/100 Iteration 870 | Training loss: 0.6861\n",
            "Epoch 5/100 Iteration 880 | Training loss: 0.7580\n",
            "Epoch 5/100 Iteration 890 | Training loss: 0.7140\n",
            "Epoch 5/100 Iteration 900 | Training loss: 0.6155\n",
            "Epoch 5/100 Iteration 910 | Training loss: 0.7136\n",
            "Epoch 5/100 Iteration 920 | Training loss: 0.6376\n",
            "Epoch 5/100 Iteration 930 | Training loss: 0.6735\n",
            "Epoch 6/100 Iteration 940 | Training loss: 0.7711\n",
            "Epoch 6/100 Iteration 950 | Training loss: 0.7803\n",
            "Epoch 6/100 Iteration 960 | Training loss: 0.6426\n",
            "Epoch 6/100 Iteration 970 | Training loss: 0.7153\n",
            "Epoch 6/100 Iteration 980 | Training loss: 0.7205\n",
            "Epoch 6/100 Iteration 990 | Training loss: 0.6278\n",
            "Epoch 6/100 Iteration 1000 | Training loss: 0.7079\n",
            "Epoch 6/100 Iteration 1010 | Training loss: 0.7121\n",
            "Epoch 6/100 Iteration 1020 | Training loss: 0.6595\n",
            "Epoch 6/100 Iteration 1030 | Training loss: 0.6112\n",
            "Epoch 6/100 Iteration 1040 | Training loss: 0.6253\n",
            "Epoch 6/100 Iteration 1050 | Training loss: 0.7279\n",
            "Epoch 6/100 Iteration 1060 | Training loss: 0.5332\n",
            "Epoch 6/100 Iteration 1070 | Training loss: 0.6259\n",
            "Epoch 6/100 Iteration 1080 | Training loss: 0.6881\n",
            "Epoch 6/100 Iteration 1090 | Training loss: 0.7228\n",
            "Epoch 6/100 Iteration 1100 | Training loss: 0.8169\n",
            "Epoch 6/100 Iteration 1110 | Training loss: 0.5478\n",
            "Epoch 7/100 Iteration 1120 | Training loss: 0.6709\n",
            "Epoch 7/100 Iteration 1130 | Training loss: 0.6293\n",
            "Epoch 7/100 Iteration 1140 | Training loss: 0.7605\n",
            "Epoch 7/100 Iteration 1150 | Training loss: 0.6910\n",
            "Epoch 7/100 Iteration 1160 | Training loss: 0.8109\n",
            "Epoch 7/100 Iteration 1170 | Training loss: 0.6337\n",
            "Epoch 7/100 Iteration 1180 | Training loss: 0.6268\n",
            "Epoch 7/100 Iteration 1190 | Training loss: 0.6453\n",
            "Epoch 7/100 Iteration 1200 | Training loss: 0.6737\n",
            "Epoch 7/100 Iteration 1210 | Training loss: 0.6884\n",
            "Epoch 7/100 Iteration 1220 | Training loss: 0.5344\n",
            "Epoch 7/100 Iteration 1230 | Training loss: 0.6197\n",
            "Epoch 7/100 Iteration 1240 | Training loss: 0.6927\n",
            "Epoch 7/100 Iteration 1250 | Training loss: 0.5852\n",
            "Epoch 7/100 Iteration 1260 | Training loss: 0.6840\n",
            "Epoch 7/100 Iteration 1270 | Training loss: 0.6097\n",
            "Epoch 7/100 Iteration 1280 | Training loss: 0.6764\n",
            "Epoch 7/100 Iteration 1290 | Training loss: 0.6922\n",
            "Epoch 7/100 Iteration 1300 | Training loss: 0.5658\n",
            "Epoch 8/100 Iteration 1310 | Training loss: 0.7086\n",
            "Epoch 8/100 Iteration 1320 | Training loss: 0.4729\n",
            "Epoch 8/100 Iteration 1330 | Training loss: 0.6607\n",
            "Epoch 8/100 Iteration 1340 | Training loss: 0.6309\n",
            "Epoch 8/100 Iteration 1350 | Training loss: 0.6473\n",
            "Epoch 8/100 Iteration 1360 | Training loss: 0.6233\n",
            "Epoch 8/100 Iteration 1370 | Training loss: 0.7176\n",
            "Epoch 8/100 Iteration 1380 | Training loss: 0.6172\n",
            "Epoch 8/100 Iteration 1390 | Training loss: 0.5328\n",
            "Epoch 8/100 Iteration 1400 | Training loss: 0.5523\n",
            "Epoch 8/100 Iteration 1410 | Training loss: 0.6461\n",
            "Epoch 8/100 Iteration 1420 | Training loss: 0.6172\n",
            "Epoch 8/100 Iteration 1430 | Training loss: 0.6331\n",
            "Epoch 8/100 Iteration 1440 | Training loss: 0.5855\n",
            "Epoch 8/100 Iteration 1450 | Training loss: 0.6636\n",
            "Epoch 8/100 Iteration 1460 | Training loss: 0.5998\n",
            "Epoch 8/100 Iteration 1470 | Training loss: 0.7579\n",
            "Epoch 8/100 Iteration 1480 | Training loss: 0.5538\n",
            "Epoch 9/100 Iteration 1490 | Training loss: 0.5481\n",
            "Epoch 9/100 Iteration 1500 | Training loss: 0.6423\n",
            "Epoch 9/100 Iteration 1510 | Training loss: 0.6809\n",
            "Epoch 9/100 Iteration 1520 | Training loss: 0.6321\n",
            "Epoch 9/100 Iteration 1530 | Training loss: 0.6653\n",
            "Epoch 9/100 Iteration 1540 | Training loss: 0.5953\n",
            "Epoch 9/100 Iteration 1550 | Training loss: 0.6415\n",
            "Epoch 9/100 Iteration 1560 | Training loss: 0.5744\n",
            "Epoch 9/100 Iteration 1570 | Training loss: 0.5809\n",
            "Epoch 9/100 Iteration 1580 | Training loss: 0.6170\n",
            "Epoch 9/100 Iteration 1590 | Training loss: 0.5284\n",
            "Epoch 9/100 Iteration 1600 | Training loss: 0.5562\n",
            "Epoch 9/100 Iteration 1610 | Training loss: 0.6094\n",
            "Epoch 9/100 Iteration 1620 | Training loss: 0.5824\n",
            "Epoch 9/100 Iteration 1630 | Training loss: 0.6068\n",
            "Epoch 9/100 Iteration 1640 | Training loss: 0.5267\n",
            "Epoch 9/100 Iteration 1650 | Training loss: 0.5836\n",
            "Epoch 9/100 Iteration 1660 | Training loss: 0.6059\n",
            "Epoch 9/100 Iteration 1670 | Training loss: 0.5969\n",
            "Epoch 10/100 Iteration 1680 | Training loss: 0.5708\n",
            "Epoch 10/100 Iteration 1690 | Training loss: 0.6178\n",
            "Epoch 10/100 Iteration 1700 | Training loss: 0.6657\n",
            "Epoch 10/100 Iteration 1710 | Training loss: 0.5766\n",
            "Epoch 10/100 Iteration 1720 | Training loss: 0.6199\n",
            "Epoch 10/100 Iteration 1730 | Training loss: 0.6321\n",
            "Epoch 10/100 Iteration 1740 | Training loss: 0.6477\n",
            "Epoch 10/100 Iteration 1750 | Training loss: 0.5343\n",
            "Epoch 10/100 Iteration 1760 | Training loss: 0.5486\n",
            "Epoch 10/100 Iteration 1770 | Training loss: 0.5526\n",
            "Epoch 10/100 Iteration 1780 | Training loss: 0.6293\n",
            "Epoch 10/100 Iteration 1790 | Training loss: 0.6628\n",
            "Epoch 10/100 Iteration 1800 | Training loss: 0.5692\n",
            "Epoch 10/100 Iteration 1810 | Training loss: 0.6478\n",
            "Epoch 10/100 Iteration 1820 | Training loss: 0.5902\n",
            "Epoch 10/100 Iteration 1830 | Training loss: 0.5202\n",
            "Epoch 10/100 Iteration 1840 | Training loss: 0.6058\n",
            "Epoch 10/100 Iteration 1850 | Training loss: 0.5338\n",
            "Epoch 10/100 Iteration 1860 | Training loss: 0.5792\n",
            "Epoch 11/100 Iteration 1870 | Training loss: 0.6532\n",
            "Epoch 11/100 Iteration 1880 | Training loss: 0.6724\n",
            "Epoch 11/100 Iteration 1890 | Training loss: 0.5438\n",
            "Epoch 11/100 Iteration 1900 | Training loss: 0.6170\n",
            "Epoch 11/100 Iteration 1910 | Training loss: 0.6201\n",
            "Epoch 11/100 Iteration 1920 | Training loss: 0.5382\n",
            "Epoch 11/100 Iteration 1930 | Training loss: 0.6182\n",
            "Epoch 11/100 Iteration 1940 | Training loss: 0.6157\n",
            "Epoch 11/100 Iteration 1950 | Training loss: 0.5638\n",
            "Epoch 11/100 Iteration 1960 | Training loss: 0.5318\n",
            "Epoch 11/100 Iteration 1970 | Training loss: 0.5497\n",
            "Epoch 11/100 Iteration 1980 | Training loss: 0.6301\n",
            "Epoch 11/100 Iteration 1990 | Training loss: 0.4661\n",
            "Epoch 11/100 Iteration 2000 | Training loss: 0.5441\n",
            "Epoch 11/100 Iteration 2010 | Training loss: 0.6077\n",
            "Epoch 11/100 Iteration 2020 | Training loss: 0.6378\n",
            "Epoch 11/100 Iteration 2030 | Training loss: 0.7223\n",
            "Epoch 11/100 Iteration 2040 | Training loss: 0.4989\n",
            "Epoch 12/100 Iteration 2050 | Training loss: 0.6244\n",
            "Epoch 12/100 Iteration 2060 | Training loss: 0.5773\n",
            "Epoch 12/100 Iteration 2070 | Training loss: 0.6854\n",
            "Epoch 12/100 Iteration 2080 | Training loss: 0.6171\n",
            "Epoch 12/100 Iteration 2090 | Training loss: 0.7299\n",
            "Epoch 12/100 Iteration 2100 | Training loss: 0.5716\n",
            "Epoch 12/100 Iteration 2110 | Training loss: 0.5699\n",
            "Epoch 12/100 Iteration 2120 | Training loss: 0.5741\n",
            "Epoch 12/100 Iteration 2130 | Training loss: 0.6042\n",
            "Epoch 12/100 Iteration 2140 | Training loss: 0.6127\n",
            "Epoch 12/100 Iteration 2150 | Training loss: 0.4749\n",
            "Epoch 12/100 Iteration 2160 | Training loss: 0.5513\n",
            "Epoch 12/100 Iteration 2170 | Training loss: 0.6223\n",
            "Epoch 12/100 Iteration 2180 | Training loss: 0.5395\n",
            "Epoch 12/100 Iteration 2190 | Training loss: 0.6205\n",
            "Epoch 12/100 Iteration 2200 | Training loss: 0.5605\n",
            "Epoch 12/100 Iteration 2210 | Training loss: 0.6039\n",
            "Epoch 12/100 Iteration 2220 | Training loss: 0.6300\n",
            "Epoch 12/100 Iteration 2230 | Training loss: 0.5162\n",
            "Epoch 13/100 Iteration 2240 | Training loss: 0.6406\n",
            "Epoch 13/100 Iteration 2250 | Training loss: 0.4269\n",
            "Epoch 13/100 Iteration 2260 | Training loss: 0.5975\n",
            "Epoch 13/100 Iteration 2270 | Training loss: 0.5819\n",
            "Epoch 13/100 Iteration 2280 | Training loss: 0.5860\n",
            "Epoch 13/100 Iteration 2290 | Training loss: 0.5680\n",
            "Epoch 13/100 Iteration 2300 | Training loss: 0.6592\n",
            "Epoch 13/100 Iteration 2310 | Training loss: 0.5580\n",
            "Epoch 13/100 Iteration 2320 | Training loss: 0.4926\n",
            "Epoch 13/100 Iteration 2330 | Training loss: 0.4989\n",
            "Epoch 13/100 Iteration 2340 | Training loss: 0.5931\n",
            "Epoch 13/100 Iteration 2350 | Training loss: 0.5715\n",
            "Epoch 13/100 Iteration 2360 | Training loss: 0.5775\n",
            "Epoch 13/100 Iteration 2370 | Training loss: 0.5381\n",
            "Epoch 13/100 Iteration 2380 | Training loss: 0.5996\n",
            "Epoch 13/100 Iteration 2390 | Training loss: 0.5481\n",
            "Epoch 13/100 Iteration 2400 | Training loss: 0.7058\n",
            "Epoch 13/100 Iteration 2410 | Training loss: 0.5152\n",
            "Epoch 14/100 Iteration 2420 | Training loss: 0.5101\n",
            "Epoch 14/100 Iteration 2430 | Training loss: 0.5965\n",
            "Epoch 14/100 Iteration 2440 | Training loss: 0.6241\n",
            "Epoch 14/100 Iteration 2450 | Training loss: 0.5832\n",
            "Epoch 14/100 Iteration 2460 | Training loss: 0.6193\n",
            "Epoch 14/100 Iteration 2470 | Training loss: 0.5432\n",
            "Epoch 14/100 Iteration 2480 | Training loss: 0.5880\n",
            "Epoch 14/100 Iteration 2490 | Training loss: 0.5312\n",
            "Epoch 14/100 Iteration 2500 | Training loss: 0.5314\n",
            "Epoch 14/100 Iteration 2510 | Training loss: 0.5690\n",
            "Epoch 14/100 Iteration 2520 | Training loss: 0.4885\n",
            "Epoch 14/100 Iteration 2530 | Training loss: 0.5082\n",
            "Epoch 14/100 Iteration 2540 | Training loss: 0.5636\n",
            "Epoch 14/100 Iteration 2550 | Training loss: 0.5359\n",
            "Epoch 14/100 Iteration 2560 | Training loss: 0.5610\n",
            "Epoch 14/100 Iteration 2570 | Training loss: 0.4886\n",
            "Epoch 14/100 Iteration 2580 | Training loss: 0.5318\n",
            "Epoch 14/100 Iteration 2590 | Training loss: 0.5586\n",
            "Epoch 14/100 Iteration 2600 | Training loss: 0.5585\n",
            "Epoch 15/100 Iteration 2610 | Training loss: 0.5311\n",
            "Epoch 15/100 Iteration 2620 | Training loss: 0.5458\n",
            "Epoch 15/100 Iteration 2630 | Training loss: 0.6107\n",
            "Epoch 15/100 Iteration 2640 | Training loss: 0.5318\n",
            "Epoch 15/100 Iteration 2650 | Training loss: 0.5724\n",
            "Epoch 15/100 Iteration 2660 | Training loss: 0.5757\n",
            "Epoch 15/100 Iteration 2670 | Training loss: 0.6060\n",
            "Epoch 15/100 Iteration 2680 | Training loss: 0.4972\n",
            "Epoch 15/100 Iteration 2690 | Training loss: 0.5130\n",
            "Epoch 15/100 Iteration 2700 | Training loss: 0.4988\n",
            "Epoch 15/100 Iteration 2710 | Training loss: 0.5767\n",
            "Epoch 15/100 Iteration 2720 | Training loss: 0.6162\n",
            "Epoch 15/100 Iteration 2730 | Training loss: 0.5335\n",
            "Epoch 15/100 Iteration 2740 | Training loss: 0.6039\n",
            "Epoch 15/100 Iteration 2750 | Training loss: 0.5507\n",
            "Epoch 15/100 Iteration 2760 | Training loss: 0.4890\n",
            "Epoch 15/100 Iteration 2770 | Training loss: 0.5645\n",
            "Epoch 15/100 Iteration 2780 | Training loss: 0.4946\n",
            "Epoch 15/100 Iteration 2790 | Training loss: 0.5419\n",
            "Epoch 16/100 Iteration 2800 | Training loss: 0.6328\n",
            "Epoch 16/100 Iteration 2810 | Training loss: 0.6396\n",
            "Epoch 16/100 Iteration 2820 | Training loss: 0.5319\n",
            "Epoch 16/100 Iteration 2830 | Training loss: 0.5981\n",
            "Epoch 16/100 Iteration 2840 | Training loss: 0.5779\n",
            "Epoch 16/100 Iteration 2850 | Training loss: 0.5083\n",
            "Epoch 16/100 Iteration 2860 | Training loss: 0.5834\n",
            "Epoch 16/100 Iteration 2870 | Training loss: 0.5841\n",
            "Epoch 16/100 Iteration 2880 | Training loss: 0.5322\n",
            "Epoch 16/100 Iteration 2890 | Training loss: 0.4972\n",
            "Epoch 16/100 Iteration 2900 | Training loss: 0.5088\n",
            "Epoch 16/100 Iteration 2910 | Training loss: 0.5958\n",
            "Epoch 16/100 Iteration 2920 | Training loss: 0.4366\n",
            "Epoch 16/100 Iteration 2930 | Training loss: 0.5134\n",
            "Epoch 16/100 Iteration 2940 | Training loss: 0.5793\n",
            "Epoch 16/100 Iteration 2950 | Training loss: 0.5904\n",
            "Epoch 16/100 Iteration 2960 | Training loss: 0.6796\n",
            "Epoch 16/100 Iteration 2970 | Training loss: 0.4696\n",
            "Epoch 17/100 Iteration 2980 | Training loss: 0.6287\n",
            "Epoch 17/100 Iteration 2990 | Training loss: 0.5686\n",
            "Epoch 17/100 Iteration 3000 | Training loss: 0.6506\n",
            "Epoch 17/100 Iteration 3010 | Training loss: 0.5864\n",
            "Epoch 17/100 Iteration 3020 | Training loss: 0.6947\n",
            "Epoch 17/100 Iteration 3030 | Training loss: 0.5395\n",
            "Epoch 17/100 Iteration 3040 | Training loss: 0.5334\n",
            "Epoch 17/100 Iteration 3050 | Training loss: 0.5463\n",
            "Epoch 17/100 Iteration 3060 | Training loss: 0.5735\n",
            "Epoch 17/100 Iteration 3070 | Training loss: 0.5914\n",
            "Epoch 17/100 Iteration 3080 | Training loss: 0.4496\n",
            "Epoch 17/100 Iteration 3090 | Training loss: 0.5281\n",
            "Epoch 17/100 Iteration 3100 | Training loss: 0.5912\n",
            "Epoch 17/100 Iteration 3110 | Training loss: 0.5069\n",
            "Epoch 17/100 Iteration 3120 | Training loss: 0.5901\n",
            "Epoch 17/100 Iteration 3130 | Training loss: 0.5218\n",
            "Epoch 17/100 Iteration 3140 | Training loss: 0.5719\n",
            "Epoch 17/100 Iteration 3150 | Training loss: 0.6056\n",
            "Epoch 17/100 Iteration 3160 | Training loss: 0.4930\n",
            "Epoch 18/100 Iteration 3170 | Training loss: 0.6220\n",
            "Epoch 18/100 Iteration 3180 | Training loss: 0.4164\n",
            "Epoch 18/100 Iteration 3190 | Training loss: 0.5664\n",
            "Epoch 18/100 Iteration 3200 | Training loss: 0.5563\n",
            "Epoch 18/100 Iteration 3210 | Training loss: 0.5528\n",
            "Epoch 18/100 Iteration 3220 | Training loss: 0.5345\n",
            "Epoch 18/100 Iteration 3230 | Training loss: 0.6245\n",
            "Epoch 18/100 Iteration 3240 | Training loss: 0.5353\n",
            "Epoch 18/100 Iteration 3250 | Training loss: 0.4700\n",
            "Epoch 18/100 Iteration 3260 | Training loss: 0.4704\n",
            "Epoch 18/100 Iteration 3270 | Training loss: 0.5680\n",
            "Epoch 18/100 Iteration 3280 | Training loss: 0.5449\n",
            "Epoch 18/100 Iteration 3290 | Training loss: 0.5449\n",
            "Epoch 18/100 Iteration 3300 | Training loss: 0.5174\n",
            "Epoch 18/100 Iteration 3310 | Training loss: 0.5684\n",
            "Epoch 18/100 Iteration 3320 | Training loss: 0.5272\n",
            "Epoch 18/100 Iteration 3330 | Training loss: 0.6689\n",
            "Epoch 18/100 Iteration 3340 | Training loss: 0.4815\n",
            "Epoch 19/100 Iteration 3350 | Training loss: 0.4811\n",
            "Epoch 19/100 Iteration 3360 | Training loss: 0.5718\n",
            "Epoch 19/100 Iteration 3370 | Training loss: 0.6026\n",
            "Epoch 19/100 Iteration 3380 | Training loss: 0.5565\n",
            "Epoch 19/100 Iteration 3390 | Training loss: 0.5976\n",
            "Epoch 19/100 Iteration 3400 | Training loss: 0.5210\n",
            "Epoch 19/100 Iteration 3410 | Training loss: 0.5620\n",
            "Epoch 19/100 Iteration 3420 | Training loss: 0.5022\n",
            "Epoch 19/100 Iteration 3430 | Training loss: 0.5042\n",
            "Epoch 19/100 Iteration 3440 | Training loss: 0.5392\n",
            "Epoch 19/100 Iteration 3450 | Training loss: 0.4613\n",
            "Epoch 19/100 Iteration 3460 | Training loss: 0.4872\n",
            "Epoch 19/100 Iteration 3470 | Training loss: 0.5409\n",
            "Epoch 19/100 Iteration 3480 | Training loss: 0.5127\n",
            "Epoch 19/100 Iteration 3490 | Training loss: 0.5278\n",
            "Epoch 19/100 Iteration 3500 | Training loss: 0.4665\n",
            "Epoch 19/100 Iteration 3510 | Training loss: 0.5198\n",
            "Epoch 19/100 Iteration 3520 | Training loss: 0.5377\n",
            "Epoch 19/100 Iteration 3530 | Training loss: 0.5321\n",
            "Epoch 20/100 Iteration 3540 | Training loss: 0.4998\n",
            "Epoch 20/100 Iteration 3550 | Training loss: 0.5183\n",
            "Epoch 20/100 Iteration 3560 | Training loss: 0.5703\n",
            "Epoch 20/100 Iteration 3570 | Training loss: 0.5092\n",
            "Epoch 20/100 Iteration 3580 | Training loss: 0.5476\n",
            "Epoch 20/100 Iteration 3590 | Training loss: 0.5544\n",
            "Epoch 20/100 Iteration 3600 | Training loss: 0.5801\n",
            "Epoch 20/100 Iteration 3610 | Training loss: 0.4769\n",
            "Epoch 20/100 Iteration 3620 | Training loss: 0.4825\n",
            "Epoch 20/100 Iteration 3630 | Training loss: 0.4761\n",
            "Epoch 20/100 Iteration 3640 | Training loss: 0.5495\n",
            "Epoch 20/100 Iteration 3650 | Training loss: 0.5940\n",
            "Epoch 20/100 Iteration 3660 | Training loss: 0.5110\n",
            "Epoch 20/100 Iteration 3670 | Training loss: 0.5688\n",
            "Epoch 20/100 Iteration 3680 | Training loss: 0.5258\n",
            "Epoch 20/100 Iteration 3690 | Training loss: 0.4702\n",
            "Epoch 20/100 Iteration 3700 | Training loss: 0.5347\n",
            "Epoch 20/100 Iteration 3710 | Training loss: 0.4682\n",
            "Epoch 20/100 Iteration 3720 | Training loss: 0.5200\n",
            "Epoch 21/100 Iteration 3730 | Training loss: 0.5763\n",
            "Epoch 21/100 Iteration 3740 | Training loss: 0.5960\n",
            "Epoch 21/100 Iteration 3750 | Training loss: 0.4907\n",
            "Epoch 21/100 Iteration 3760 | Training loss: 0.5534\n",
            "Epoch 21/100 Iteration 3770 | Training loss: 0.5481\n",
            "Epoch 21/100 Iteration 3780 | Training loss: 0.4784\n",
            "Epoch 21/100 Iteration 3790 | Training loss: 0.5590\n",
            "Epoch 21/100 Iteration 3800 | Training loss: 0.5556\n",
            "Epoch 21/100 Iteration 3810 | Training loss: 0.5102\n",
            "Epoch 21/100 Iteration 3820 | Training loss: 0.4767\n",
            "Epoch 21/100 Iteration 3830 | Training loss: 0.4853\n",
            "Epoch 21/100 Iteration 3840 | Training loss: 0.5717\n",
            "Epoch 21/100 Iteration 3850 | Training loss: 0.4155\n",
            "Epoch 21/100 Iteration 3860 | Training loss: 0.4908\n",
            "Epoch 21/100 Iteration 3870 | Training loss: 0.5489\n",
            "Epoch 21/100 Iteration 3880 | Training loss: 0.5625\n",
            "Epoch 21/100 Iteration 3890 | Training loss: 0.6551\n",
            "Epoch 21/100 Iteration 3900 | Training loss: 0.4502\n",
            "Epoch 22/100 Iteration 3910 | Training loss: 0.5317\n",
            "Epoch 22/100 Iteration 3920 | Training loss: 0.5027\n",
            "Epoch 22/100 Iteration 3930 | Training loss: 0.6001\n",
            "Epoch 22/100 Iteration 3940 | Training loss: 0.5532\n",
            "Epoch 22/100 Iteration 3950 | Training loss: 0.6512\n",
            "Epoch 22/100 Iteration 3960 | Training loss: 0.5124\n",
            "Epoch 22/100 Iteration 3970 | Training loss: 0.5089\n",
            "Epoch 22/100 Iteration 3980 | Training loss: 0.5088\n",
            "Epoch 22/100 Iteration 3990 | Training loss: 0.5401\n",
            "Epoch 22/100 Iteration 4000 | Training loss: 0.5511\n",
            "Epoch 22/100 Iteration 4010 | Training loss: 0.4279\n",
            "Epoch 22/100 Iteration 4020 | Training loss: 0.5002\n",
            "Epoch 22/100 Iteration 4030 | Training loss: 0.5599\n",
            "Epoch 22/100 Iteration 4040 | Training loss: 0.4843\n",
            "Epoch 22/100 Iteration 4050 | Training loss: 0.5609\n",
            "Epoch 22/100 Iteration 4060 | Training loss: 0.5021\n",
            "Epoch 22/100 Iteration 4070 | Training loss: 0.5512\n",
            "Epoch 22/100 Iteration 4080 | Training loss: 0.5791\n",
            "Epoch 22/100 Iteration 4090 | Training loss: 0.4820\n",
            "Epoch 23/100 Iteration 4100 | Training loss: 0.5825\n",
            "Epoch 23/100 Iteration 4110 | Training loss: 0.3930\n",
            "Epoch 23/100 Iteration 4120 | Training loss: 0.5341\n",
            "Epoch 23/100 Iteration 4130 | Training loss: 0.5360\n",
            "Epoch 23/100 Iteration 4140 | Training loss: 0.5267\n",
            "Epoch 23/100 Iteration 4150 | Training loss: 0.5114\n",
            "Epoch 23/100 Iteration 4160 | Training loss: 0.5967\n",
            "Epoch 23/100 Iteration 4170 | Training loss: 0.5074\n",
            "Epoch 23/100 Iteration 4180 | Training loss: 0.4446\n",
            "Epoch 23/100 Iteration 4190 | Training loss: 0.4532\n",
            "Epoch 23/100 Iteration 4200 | Training loss: 0.5511\n",
            "Epoch 23/100 Iteration 4210 | Training loss: 0.5232\n",
            "Epoch 23/100 Iteration 4220 | Training loss: 0.5279\n",
            "Epoch 23/100 Iteration 4230 | Training loss: 0.4912\n",
            "Epoch 23/100 Iteration 4240 | Training loss: 0.5421\n",
            "Epoch 23/100 Iteration 4250 | Training loss: 0.5122\n",
            "Epoch 23/100 Iteration 4260 | Training loss: 0.6532\n",
            "Epoch 23/100 Iteration 4270 | Training loss: 0.4655\n",
            "Epoch 24/100 Iteration 4280 | Training loss: 0.4637\n",
            "Epoch 24/100 Iteration 4290 | Training loss: 0.5448\n",
            "Epoch 24/100 Iteration 4300 | Training loss: 0.5732\n",
            "Epoch 24/100 Iteration 4310 | Training loss: 0.5374\n",
            "Epoch 24/100 Iteration 4320 | Training loss: 0.5776\n",
            "Epoch 24/100 Iteration 4330 | Training loss: 0.4942\n",
            "Epoch 24/100 Iteration 4340 | Training loss: 0.5355\n",
            "Epoch 24/100 Iteration 4350 | Training loss: 0.4755\n",
            "Epoch 24/100 Iteration 4360 | Training loss: 0.4805\n",
            "Epoch 24/100 Iteration 4370 | Training loss: 0.5125\n",
            "Epoch 24/100 Iteration 4380 | Training loss: 0.4432\n",
            "Epoch 24/100 Iteration 4390 | Training loss: 0.4611\n",
            "Epoch 24/100 Iteration 4400 | Training loss: 0.5145\n",
            "Epoch 24/100 Iteration 4410 | Training loss: 0.4955\n",
            "Epoch 24/100 Iteration 4420 | Training loss: 0.5093\n",
            "Epoch 24/100 Iteration 4430 | Training loss: 0.4523\n",
            "Epoch 24/100 Iteration 4440 | Training loss: 0.4923\n",
            "Epoch 24/100 Iteration 4450 | Training loss: 0.5050\n",
            "Epoch 24/100 Iteration 4460 | Training loss: 0.5030\n",
            "Epoch 25/100 Iteration 4470 | Training loss: 0.4806\n",
            "Epoch 25/100 Iteration 4480 | Training loss: 0.4923\n",
            "Epoch 25/100 Iteration 4490 | Training loss: 0.5542\n",
            "Epoch 25/100 Iteration 4500 | Training loss: 0.4907\n",
            "Epoch 25/100 Iteration 4510 | Training loss: 0.5225\n",
            "Epoch 25/100 Iteration 4520 | Training loss: 0.5226\n",
            "Epoch 25/100 Iteration 4530 | Training loss: 0.5560\n",
            "Epoch 25/100 Iteration 4540 | Training loss: 0.4567\n",
            "Epoch 25/100 Iteration 4550 | Training loss: 0.4725\n",
            "Epoch 25/100 Iteration 4560 | Training loss: 0.4562\n",
            "Epoch 25/100 Iteration 4570 | Training loss: 0.5339\n",
            "Epoch 25/100 Iteration 4580 | Training loss: 0.5721\n",
            "Epoch 25/100 Iteration 4590 | Training loss: 0.4898\n",
            "Epoch 25/100 Iteration 4600 | Training loss: 0.5501\n",
            "Epoch 25/100 Iteration 4610 | Training loss: 0.5107\n",
            "Epoch 25/100 Iteration 4620 | Training loss: 0.4440\n",
            "Epoch 25/100 Iteration 4630 | Training loss: 0.5240\n",
            "Epoch 25/100 Iteration 4640 | Training loss: 0.4563\n",
            "Epoch 25/100 Iteration 4650 | Training loss: 0.5074\n",
            "Epoch 26/100 Iteration 4660 | Training loss: 0.5507\n",
            "Epoch 26/100 Iteration 4670 | Training loss: 0.5737\n",
            "Epoch 26/100 Iteration 4680 | Training loss: 0.4728\n",
            "Epoch 26/100 Iteration 4690 | Training loss: 0.5312\n",
            "Epoch 26/100 Iteration 4700 | Training loss: 0.5248\n",
            "Epoch 26/100 Iteration 4710 | Training loss: 0.4537\n",
            "Epoch 26/100 Iteration 4720 | Training loss: 0.5379\n",
            "Epoch 26/100 Iteration 4730 | Training loss: 0.5361\n",
            "Epoch 26/100 Iteration 4740 | Training loss: 0.4823\n",
            "Epoch 26/100 Iteration 4750 | Training loss: 0.4541\n",
            "Epoch 26/100 Iteration 4760 | Training loss: 0.4537\n",
            "Epoch 26/100 Iteration 4770 | Training loss: 0.5435\n",
            "Epoch 26/100 Iteration 4780 | Training loss: 0.3972\n",
            "Epoch 26/100 Iteration 4790 | Training loss: 0.4776\n",
            "Epoch 26/100 Iteration 4800 | Training loss: 0.5319\n",
            "Epoch 26/100 Iteration 4810 | Training loss: 0.5474\n",
            "Epoch 26/100 Iteration 4820 | Training loss: 0.6339\n",
            "Epoch 26/100 Iteration 4830 | Training loss: 0.4338\n",
            "Epoch 27/100 Iteration 4840 | Training loss: 0.5064\n",
            "Epoch 27/100 Iteration 4850 | Training loss: 0.4859\n",
            "Epoch 27/100 Iteration 4860 | Training loss: 0.5754\n",
            "Epoch 27/100 Iteration 4870 | Training loss: 0.5334\n",
            "Epoch 27/100 Iteration 4880 | Training loss: 0.6353\n",
            "Epoch 27/100 Iteration 4890 | Training loss: 0.4871\n",
            "Epoch 27/100 Iteration 4900 | Training loss: 0.4849\n",
            "Epoch 27/100 Iteration 4910 | Training loss: 0.4881\n",
            "Epoch 27/100 Iteration 4920 | Training loss: 0.5222\n",
            "Epoch 27/100 Iteration 4930 | Training loss: 0.5400\n",
            "Epoch 27/100 Iteration 4940 | Training loss: 0.4142\n",
            "Epoch 27/100 Iteration 4950 | Training loss: 0.4792\n",
            "Epoch 27/100 Iteration 4960 | Training loss: 0.5369\n",
            "Epoch 27/100 Iteration 4970 | Training loss: 0.4665\n",
            "Epoch 27/100 Iteration 4980 | Training loss: 0.5444\n",
            "Epoch 27/100 Iteration 4990 | Training loss: 0.4820\n",
            "Epoch 27/100 Iteration 5000 | Training loss: 0.5278\n",
            "Epoch 27/100 Iteration 5010 | Training loss: 0.5584\n",
            "Epoch 27/100 Iteration 5020 | Training loss: 0.4661\n",
            "Epoch 28/100 Iteration 5030 | Training loss: 0.5654\n",
            "Epoch 28/100 Iteration 5040 | Training loss: 0.3863\n",
            "Epoch 28/100 Iteration 5050 | Training loss: 0.5139\n",
            "Epoch 28/100 Iteration 5060 | Training loss: 0.5210\n",
            "Epoch 28/100 Iteration 5070 | Training loss: 0.5066\n",
            "Epoch 28/100 Iteration 5080 | Training loss: 0.4953\n",
            "Epoch 28/100 Iteration 5090 | Training loss: 0.5755\n",
            "Epoch 28/100 Iteration 5100 | Training loss: 0.4925\n",
            "Epoch 28/100 Iteration 5110 | Training loss: 0.4265\n",
            "Epoch 28/100 Iteration 5120 | Training loss: 0.4305\n",
            "Epoch 28/100 Iteration 5130 | Training loss: 0.5296\n",
            "Epoch 28/100 Iteration 5140 | Training loss: 0.5027\n",
            "Epoch 28/100 Iteration 5150 | Training loss: 0.5148\n",
            "Epoch 28/100 Iteration 5160 | Training loss: 0.4746\n",
            "Epoch 28/100 Iteration 5170 | Training loss: 0.5280\n",
            "Epoch 28/100 Iteration 5180 | Training loss: 0.4855\n",
            "Epoch 28/100 Iteration 5190 | Training loss: 0.6241\n",
            "Epoch 28/100 Iteration 5200 | Training loss: 0.4464\n",
            "Epoch 29/100 Iteration 5210 | Training loss: 0.4458\n",
            "Epoch 29/100 Iteration 5220 | Training loss: 0.5317\n",
            "Epoch 29/100 Iteration 5230 | Training loss: 0.5520\n",
            "Epoch 29/100 Iteration 5240 | Training loss: 0.5214\n",
            "Epoch 29/100 Iteration 5250 | Training loss: 0.5615\n",
            "Epoch 29/100 Iteration 5260 | Training loss: 0.4744\n",
            "Epoch 29/100 Iteration 5270 | Training loss: 0.5211\n",
            "Epoch 29/100 Iteration 5280 | Training loss: 0.4616\n",
            "Epoch 29/100 Iteration 5290 | Training loss: 0.4595\n",
            "Epoch 29/100 Iteration 5300 | Training loss: 0.4951\n",
            "Epoch 29/100 Iteration 5310 | Training loss: 0.4239\n",
            "Epoch 29/100 Iteration 5320 | Training loss: 0.4446\n",
            "Epoch 29/100 Iteration 5330 | Training loss: 0.5006\n",
            "Epoch 29/100 Iteration 5340 | Training loss: 0.4801\n",
            "Epoch 29/100 Iteration 5350 | Training loss: 0.4955\n",
            "Epoch 29/100 Iteration 5360 | Training loss: 0.4335\n",
            "Epoch 29/100 Iteration 5370 | Training loss: 0.4792\n",
            "Epoch 29/100 Iteration 5380 | Training loss: 0.4979\n",
            "Epoch 29/100 Iteration 5390 | Training loss: 0.4861\n",
            "Epoch 30/100 Iteration 5400 | Training loss: 0.4675\n",
            "Epoch 30/100 Iteration 5410 | Training loss: 0.4806\n",
            "Epoch 30/100 Iteration 5420 | Training loss: 0.5558\n",
            "Epoch 30/100 Iteration 5430 | Training loss: 0.4849\n",
            "Epoch 30/100 Iteration 5440 | Training loss: 0.5202\n",
            "Epoch 30/100 Iteration 5450 | Training loss: 0.5160\n",
            "Epoch 30/100 Iteration 5460 | Training loss: 0.5420\n",
            "Epoch 30/100 Iteration 5470 | Training loss: 0.4482\n",
            "Epoch 30/100 Iteration 5480 | Training loss: 0.4499\n",
            "Epoch 30/100 Iteration 5490 | Training loss: 0.4413\n",
            "Epoch 30/100 Iteration 5500 | Training loss: 0.5122\n",
            "Epoch 30/100 Iteration 5510 | Training loss: 0.5511\n",
            "Epoch 30/100 Iteration 5520 | Training loss: 0.4729\n",
            "Epoch 30/100 Iteration 5530 | Training loss: 0.5313\n",
            "Epoch 30/100 Iteration 5540 | Training loss: 0.4911\n",
            "Epoch 30/100 Iteration 5550 | Training loss: 0.4312\n",
            "Epoch 30/100 Iteration 5560 | Training loss: 0.5036\n",
            "Epoch 30/100 Iteration 5570 | Training loss: 0.4327\n",
            "Epoch 30/100 Iteration 5580 | Training loss: 0.4895\n",
            "Epoch 31/100 Iteration 5590 | Training loss: 0.5316\n",
            "Epoch 31/100 Iteration 5600 | Training loss: 0.5539\n",
            "Epoch 31/100 Iteration 5610 | Training loss: 0.4567\n",
            "Epoch 31/100 Iteration 5620 | Training loss: 0.5115\n",
            "Epoch 31/100 Iteration 5630 | Training loss: 0.5105\n",
            "Epoch 31/100 Iteration 5640 | Training loss: 0.4369\n",
            "Epoch 31/100 Iteration 5650 | Training loss: 0.5150\n",
            "Epoch 31/100 Iteration 5660 | Training loss: 0.5108\n",
            "Epoch 31/100 Iteration 5670 | Training loss: 0.4594\n",
            "Epoch 31/100 Iteration 5680 | Training loss: 0.4414\n",
            "Epoch 31/100 Iteration 5690 | Training loss: 0.4329\n",
            "Epoch 31/100 Iteration 5700 | Training loss: 0.5307\n",
            "Epoch 31/100 Iteration 5710 | Training loss: 0.3817\n",
            "Epoch 31/100 Iteration 5720 | Training loss: 0.4578\n",
            "Epoch 31/100 Iteration 5730 | Training loss: 0.5080\n",
            "Epoch 31/100 Iteration 5740 | Training loss: 0.5206\n",
            "Epoch 31/100 Iteration 5750 | Training loss: 0.6072\n",
            "Epoch 31/100 Iteration 5760 | Training loss: 0.4297\n",
            "Epoch 32/100 Iteration 5770 | Training loss: 0.4925\n",
            "Epoch 32/100 Iteration 5780 | Training loss: 0.4616\n",
            "Epoch 32/100 Iteration 5790 | Training loss: 0.5558\n",
            "Epoch 32/100 Iteration 5800 | Training loss: 0.5117\n",
            "Epoch 32/100 Iteration 5810 | Training loss: 0.6232\n",
            "Epoch 32/100 Iteration 5820 | Training loss: 0.4773\n",
            "Epoch 32/100 Iteration 5830 | Training loss: 0.4707\n",
            "Epoch 32/100 Iteration 5840 | Training loss: 0.4825\n",
            "Epoch 32/100 Iteration 5850 | Training loss: 0.5000\n",
            "Epoch 32/100 Iteration 5860 | Training loss: 0.5264\n",
            "Epoch 32/100 Iteration 5870 | Training loss: 0.3970\n",
            "Epoch 32/100 Iteration 5880 | Training loss: 0.4599\n",
            "Epoch 32/100 Iteration 5890 | Training loss: 0.5124\n",
            "Epoch 32/100 Iteration 5900 | Training loss: 0.4665\n",
            "Epoch 32/100 Iteration 5910 | Training loss: 0.5804\n",
            "Epoch 32/100 Iteration 5920 | Training loss: 0.4698\n",
            "Epoch 32/100 Iteration 5930 | Training loss: 0.5196\n",
            "Epoch 32/100 Iteration 5940 | Training loss: 0.5462\n",
            "Epoch 32/100 Iteration 5950 | Training loss: 0.4509\n",
            "Epoch 33/100 Iteration 5960 | Training loss: 0.5453\n",
            "Epoch 33/100 Iteration 5970 | Training loss: 0.3588\n",
            "Epoch 33/100 Iteration 5980 | Training loss: 0.5023\n",
            "Epoch 33/100 Iteration 5990 | Training loss: 0.5046\n",
            "Epoch 33/100 Iteration 6000 | Training loss: 0.4900\n",
            "Epoch 33/100 Iteration 6010 | Training loss: 0.4775\n",
            "Epoch 33/100 Iteration 6020 | Training loss: 0.5579\n",
            "Epoch 33/100 Iteration 6030 | Training loss: 0.4782\n",
            "Epoch 33/100 Iteration 6040 | Training loss: 0.4055\n",
            "Epoch 33/100 Iteration 6050 | Training loss: 0.4194\n",
            "Epoch 33/100 Iteration 6060 | Training loss: 0.5123\n",
            "Epoch 33/100 Iteration 6070 | Training loss: 0.4857\n",
            "Epoch 33/100 Iteration 6080 | Training loss: 0.4961\n",
            "Epoch 33/100 Iteration 6090 | Training loss: 0.4606\n",
            "Epoch 33/100 Iteration 6100 | Training loss: 0.5052\n",
            "Epoch 33/100 Iteration 6110 | Training loss: 0.4727\n",
            "Epoch 33/100 Iteration 6120 | Training loss: 0.6070\n",
            "Epoch 33/100 Iteration 6130 | Training loss: 0.4374\n",
            "Epoch 34/100 Iteration 6140 | Training loss: 0.4349\n",
            "Epoch 34/100 Iteration 6150 | Training loss: 0.5326\n",
            "Epoch 34/100 Iteration 6160 | Training loss: 0.5439\n",
            "Epoch 34/100 Iteration 6170 | Training loss: 0.5085\n",
            "Epoch 34/100 Iteration 6180 | Training loss: 0.5503\n",
            "Epoch 34/100 Iteration 6190 | Training loss: 0.4583\n",
            "Epoch 34/100 Iteration 6200 | Training loss: 0.5114\n",
            "Epoch 34/100 Iteration 6210 | Training loss: 0.4520\n",
            "Epoch 34/100 Iteration 6220 | Training loss: 0.4527\n",
            "Epoch 34/100 Iteration 6230 | Training loss: 0.4806\n",
            "Epoch 34/100 Iteration 6240 | Training loss: 0.4157\n",
            "Epoch 34/100 Iteration 6250 | Training loss: 0.4346\n",
            "Epoch 34/100 Iteration 6260 | Training loss: 0.4881\n",
            "Epoch 34/100 Iteration 6270 | Training loss: 0.4572\n",
            "Epoch 34/100 Iteration 6280 | Training loss: 0.4789\n",
            "Epoch 34/100 Iteration 6290 | Training loss: 0.4188\n",
            "Epoch 34/100 Iteration 6300 | Training loss: 0.4638\n",
            "Epoch 34/100 Iteration 6310 | Training loss: 0.4856\n",
            "Epoch 34/100 Iteration 6320 | Training loss: 0.4717\n",
            "Epoch 35/100 Iteration 6330 | Training loss: 0.4629\n",
            "Epoch 35/100 Iteration 6340 | Training loss: 0.4685\n",
            "Epoch 35/100 Iteration 6350 | Training loss: 0.5323\n",
            "Epoch 35/100 Iteration 6360 | Training loss: 0.4617\n",
            "Epoch 35/100 Iteration 6370 | Training loss: 0.5014\n",
            "Epoch 35/100 Iteration 6380 | Training loss: 0.4896\n",
            "Epoch 35/100 Iteration 6390 | Training loss: 0.5197\n",
            "Epoch 35/100 Iteration 6400 | Training loss: 0.4236\n",
            "Epoch 35/100 Iteration 6410 | Training loss: 0.4286\n",
            "Epoch 35/100 Iteration 6420 | Training loss: 0.4307\n",
            "Epoch 35/100 Iteration 6430 | Training loss: 0.4845\n",
            "Epoch 35/100 Iteration 6440 | Training loss: 0.5283\n",
            "Epoch 35/100 Iteration 6450 | Training loss: 0.4569\n",
            "Epoch 35/100 Iteration 6460 | Training loss: 0.5208\n",
            "Epoch 35/100 Iteration 6470 | Training loss: 0.4780\n",
            "Epoch 35/100 Iteration 6480 | Training loss: 0.4152\n",
            "Epoch 35/100 Iteration 6490 | Training loss: 0.4938\n",
            "Epoch 35/100 Iteration 6500 | Training loss: 0.4130\n",
            "Epoch 35/100 Iteration 6510 | Training loss: 0.4821\n",
            "Epoch 36/100 Iteration 6520 | Training loss: 0.5161\n",
            "Epoch 36/100 Iteration 6530 | Training loss: 0.5385\n",
            "Epoch 36/100 Iteration 6540 | Training loss: 0.4477\n",
            "Epoch 36/100 Iteration 6550 | Training loss: 0.5035\n",
            "Epoch 36/100 Iteration 6560 | Training loss: 0.4924\n",
            "Epoch 36/100 Iteration 6570 | Training loss: 0.4281\n",
            "Epoch 36/100 Iteration 6580 | Training loss: 0.5089\n",
            "Epoch 36/100 Iteration 6590 | Training loss: 0.4999\n",
            "Epoch 36/100 Iteration 6600 | Training loss: 0.4486\n",
            "Epoch 36/100 Iteration 6610 | Training loss: 0.4341\n",
            "Epoch 36/100 Iteration 6620 | Training loss: 0.4176\n",
            "Epoch 36/100 Iteration 6630 | Training loss: 0.5152\n",
            "Epoch 36/100 Iteration 6640 | Training loss: 0.3693\n",
            "Epoch 36/100 Iteration 6650 | Training loss: 0.4401\n",
            "Epoch 36/100 Iteration 6660 | Training loss: 0.4992\n",
            "Epoch 36/100 Iteration 6670 | Training loss: 0.5122\n",
            "Epoch 36/100 Iteration 6680 | Training loss: 0.5979\n",
            "Epoch 36/100 Iteration 6690 | Training loss: 0.4163\n",
            "Epoch 37/100 Iteration 6700 | Training loss: 0.4726\n",
            "Epoch 37/100 Iteration 6710 | Training loss: 0.4551\n",
            "Epoch 37/100 Iteration 6720 | Training loss: 0.5415\n",
            "Epoch 37/100 Iteration 6730 | Training loss: 0.4957\n",
            "Epoch 37/100 Iteration 6740 | Training loss: 0.6042\n",
            "Epoch 37/100 Iteration 6750 | Training loss: 0.4582\n",
            "Epoch 37/100 Iteration 6760 | Training loss: 0.4546\n",
            "Epoch 37/100 Iteration 6770 | Training loss: 0.4643\n",
            "Epoch 37/100 Iteration 6780 | Training loss: 0.4906\n",
            "Epoch 37/100 Iteration 6790 | Training loss: 0.5117\n",
            "Epoch 37/100 Iteration 6800 | Training loss: 0.3888\n",
            "Epoch 37/100 Iteration 6810 | Training loss: 0.4554\n",
            "Epoch 37/100 Iteration 6820 | Training loss: 0.4934\n",
            "Epoch 37/100 Iteration 6830 | Training loss: 0.4432\n",
            "Epoch 37/100 Iteration 6840 | Training loss: 0.5204\n",
            "Epoch 37/100 Iteration 6850 | Training loss: 0.4416\n",
            "Epoch 37/100 Iteration 6860 | Training loss: 0.4968\n",
            "Epoch 37/100 Iteration 6870 | Training loss: 0.5222\n",
            "Epoch 37/100 Iteration 6880 | Training loss: 0.4371\n",
            "Epoch 38/100 Iteration 6890 | Training loss: 0.5363\n",
            "Epoch 38/100 Iteration 6900 | Training loss: 0.3471\n",
            "Epoch 38/100 Iteration 6910 | Training loss: 0.4801\n",
            "Epoch 38/100 Iteration 6920 | Training loss: 0.4901\n",
            "Epoch 38/100 Iteration 6930 | Training loss: 0.4743\n",
            "Epoch 38/100 Iteration 6940 | Training loss: 0.4662\n",
            "Epoch 38/100 Iteration 6950 | Training loss: 0.5507\n",
            "Epoch 38/100 Iteration 6960 | Training loss: 0.4626\n",
            "Epoch 38/100 Iteration 6970 | Training loss: 0.3910\n",
            "Epoch 38/100 Iteration 6980 | Training loss: 0.4058\n",
            "Epoch 38/100 Iteration 6990 | Training loss: 0.4969\n",
            "Epoch 38/100 Iteration 7000 | Training loss: 0.4689\n",
            "Epoch 38/100 Iteration 7010 | Training loss: 0.4868\n",
            "Epoch 38/100 Iteration 7020 | Training loss: 0.4456\n",
            "Epoch 38/100 Iteration 7030 | Training loss: 0.4964\n",
            "Epoch 38/100 Iteration 7040 | Training loss: 0.4640\n",
            "Epoch 38/100 Iteration 7050 | Training loss: 0.5943\n",
            "Epoch 38/100 Iteration 7060 | Training loss: 0.4268\n",
            "Epoch 39/100 Iteration 7070 | Training loss: 0.4211\n",
            "Epoch 39/100 Iteration 7080 | Training loss: 0.4962\n",
            "Epoch 39/100 Iteration 7090 | Training loss: 0.5239\n",
            "Epoch 39/100 Iteration 7100 | Training loss: 0.4971\n",
            "Epoch 39/100 Iteration 7110 | Training loss: 0.5355\n",
            "Epoch 39/100 Iteration 7120 | Training loss: 0.4441\n",
            "Epoch 39/100 Iteration 7130 | Training loss: 0.4939\n",
            "Epoch 39/100 Iteration 7140 | Training loss: 0.4360\n",
            "Epoch 39/100 Iteration 7150 | Training loss: 0.4244\n",
            "Epoch 39/100 Iteration 7160 | Training loss: 0.4740\n",
            "Epoch 39/100 Iteration 7170 | Training loss: 0.4020\n",
            "Epoch 39/100 Iteration 7180 | Training loss: 0.4136\n",
            "Epoch 39/100 Iteration 7190 | Training loss: 0.4659\n",
            "Epoch 39/100 Iteration 7200 | Training loss: 0.4466\n",
            "Epoch 39/100 Iteration 7210 | Training loss: 0.4582\n",
            "Epoch 39/100 Iteration 7220 | Training loss: 0.4048\n",
            "Epoch 39/100 Iteration 7230 | Training loss: 0.4522\n",
            "Epoch 39/100 Iteration 7240 | Training loss: 0.4756\n",
            "Epoch 39/100 Iteration 7250 | Training loss: 0.4570\n",
            "Epoch 40/100 Iteration 7260 | Training loss: 0.4441\n",
            "Epoch 40/100 Iteration 7270 | Training loss: 0.4545\n",
            "Epoch 40/100 Iteration 7280 | Training loss: 0.5127\n",
            "Epoch 40/100 Iteration 7290 | Training loss: 0.4492\n",
            "Epoch 40/100 Iteration 7300 | Training loss: 0.4843\n",
            "Epoch 40/100 Iteration 7310 | Training loss: 0.4735\n",
            "Epoch 40/100 Iteration 7320 | Training loss: 0.5070\n",
            "Epoch 40/100 Iteration 7330 | Training loss: 0.4138\n",
            "Epoch 40/100 Iteration 7340 | Training loss: 0.4217\n",
            "Epoch 40/100 Iteration 7350 | Training loss: 0.4221\n",
            "Epoch 40/100 Iteration 7360 | Training loss: 0.4733\n",
            "Epoch 40/100 Iteration 7370 | Training loss: 0.5229\n",
            "Epoch 40/100 Iteration 7380 | Training loss: 0.4505\n",
            "Epoch 40/100 Iteration 7390 | Training loss: 0.5043\n",
            "Epoch 40/100 Iteration 7400 | Training loss: 0.4677\n",
            "Epoch 40/100 Iteration 7410 | Training loss: 0.4135\n",
            "Epoch 40/100 Iteration 7420 | Training loss: 0.4767\n",
            "Epoch 40/100 Iteration 7430 | Training loss: 0.4030\n",
            "Epoch 40/100 Iteration 7440 | Training loss: 0.4673\n",
            "Epoch 41/100 Iteration 7450 | Training loss: 0.5023\n",
            "Epoch 41/100 Iteration 7460 | Training loss: 0.5248\n",
            "Epoch 41/100 Iteration 7470 | Training loss: 0.4244\n",
            "Epoch 41/100 Iteration 7480 | Training loss: 0.4848\n",
            "Epoch 41/100 Iteration 7490 | Training loss: 0.4817\n",
            "Epoch 41/100 Iteration 7500 | Training loss: 0.4135\n",
            "Epoch 41/100 Iteration 7510 | Training loss: 0.4930\n",
            "Epoch 41/100 Iteration 7520 | Training loss: 0.4904\n",
            "Epoch 41/100 Iteration 7530 | Training loss: 0.4384\n",
            "Epoch 41/100 Iteration 7540 | Training loss: 0.4221\n",
            "Epoch 41/100 Iteration 7550 | Training loss: 0.4018\n",
            "Epoch 41/100 Iteration 7560 | Training loss: 0.5011\n",
            "Epoch 41/100 Iteration 7570 | Training loss: 0.3559\n",
            "Epoch 41/100 Iteration 7580 | Training loss: 0.4325\n",
            "Epoch 41/100 Iteration 7590 | Training loss: 0.4828\n",
            "Epoch 41/100 Iteration 7600 | Training loss: 0.4918\n",
            "Epoch 41/100 Iteration 7610 | Training loss: 0.5818\n",
            "Epoch 41/100 Iteration 7620 | Training loss: 0.3986\n",
            "Epoch 42/100 Iteration 7630 | Training loss: 0.4691\n",
            "Epoch 42/100 Iteration 7640 | Training loss: 0.4460\n",
            "Epoch 42/100 Iteration 7650 | Training loss: 0.5308\n",
            "Epoch 42/100 Iteration 7660 | Training loss: 0.4922\n",
            "Epoch 42/100 Iteration 7670 | Training loss: 0.5930\n",
            "Epoch 42/100 Iteration 7680 | Training loss: 0.4519\n",
            "Epoch 42/100 Iteration 7690 | Training loss: 0.4361\n",
            "Epoch 42/100 Iteration 7700 | Training loss: 0.4583\n",
            "Epoch 42/100 Iteration 7710 | Training loss: 0.4656\n",
            "Epoch 42/100 Iteration 7720 | Training loss: 0.4959\n",
            "Epoch 42/100 Iteration 7730 | Training loss: 0.3766\n",
            "Epoch 42/100 Iteration 7740 | Training loss: 0.4374\n",
            "Epoch 42/100 Iteration 7750 | Training loss: 0.4878\n",
            "Epoch 42/100 Iteration 7760 | Training loss: 0.4375\n",
            "Epoch 42/100 Iteration 7770 | Training loss: 0.5127\n",
            "Epoch 42/100 Iteration 7780 | Training loss: 0.4329\n",
            "Epoch 42/100 Iteration 7790 | Training loss: 0.4865\n",
            "Epoch 42/100 Iteration 7800 | Training loss: 0.5161\n",
            "Epoch 42/100 Iteration 7810 | Training loss: 0.4285\n",
            "Epoch 43/100 Iteration 7820 | Training loss: 0.5219\n",
            "Epoch 43/100 Iteration 7830 | Training loss: 0.3373\n",
            "Epoch 43/100 Iteration 7840 | Training loss: 0.4691\n",
            "Epoch 43/100 Iteration 7850 | Training loss: 0.4801\n",
            "Epoch 43/100 Iteration 7860 | Training loss: 0.4576\n",
            "Epoch 43/100 Iteration 7870 | Training loss: 0.4597\n",
            "Epoch 43/100 Iteration 7880 | Training loss: 0.5304\n",
            "Epoch 43/100 Iteration 7890 | Training loss: 0.4498\n",
            "Epoch 43/100 Iteration 7900 | Training loss: 0.3836\n",
            "Epoch 43/100 Iteration 7910 | Training loss: 0.3926\n",
            "Epoch 43/100 Iteration 7920 | Training loss: 0.4861\n",
            "Epoch 43/100 Iteration 7930 | Training loss: 0.4567\n",
            "Epoch 43/100 Iteration 7940 | Training loss: 0.4755\n",
            "Epoch 43/100 Iteration 7950 | Training loss: 0.4323\n",
            "Epoch 43/100 Iteration 7960 | Training loss: 0.4884\n",
            "Epoch 43/100 Iteration 7970 | Training loss: 0.4524\n",
            "Epoch 43/100 Iteration 7980 | Training loss: 0.5772\n",
            "Epoch 43/100 Iteration 7990 | Training loss: 0.4088\n",
            "Epoch 44/100 Iteration 8000 | Training loss: 0.4147\n",
            "Epoch 44/100 Iteration 8010 | Training loss: 0.4872\n",
            "Epoch 44/100 Iteration 8020 | Training loss: 0.5126\n",
            "Epoch 44/100 Iteration 8030 | Training loss: 0.4820\n",
            "Epoch 44/100 Iteration 8040 | Training loss: 0.5256\n",
            "Epoch 44/100 Iteration 8050 | Training loss: 0.4286\n",
            "Epoch 44/100 Iteration 8060 | Training loss: 0.4807\n",
            "Epoch 44/100 Iteration 8070 | Training loss: 0.4209\n",
            "Epoch 44/100 Iteration 8080 | Training loss: 0.4163\n",
            "Epoch 44/100 Iteration 8090 | Training loss: 0.4637\n",
            "Epoch 44/100 Iteration 8100 | Training loss: 0.3964\n",
            "Epoch 44/100 Iteration 8110 | Training loss: 0.4027\n",
            "Epoch 44/100 Iteration 8120 | Training loss: 0.4648\n",
            "Epoch 44/100 Iteration 8130 | Training loss: 0.4366\n",
            "Epoch 44/100 Iteration 8140 | Training loss: 0.4458\n",
            "Epoch 44/100 Iteration 8150 | Training loss: 0.3967\n",
            "Epoch 44/100 Iteration 8160 | Training loss: 0.4347\n",
            "Epoch 44/100 Iteration 8170 | Training loss: 0.4597\n",
            "Epoch 44/100 Iteration 8180 | Training loss: 0.4456\n",
            "Epoch 45/100 Iteration 8190 | Training loss: 0.4420\n",
            "Epoch 45/100 Iteration 8200 | Training loss: 0.4396\n",
            "Epoch 45/100 Iteration 8210 | Training loss: 0.5046\n",
            "Epoch 45/100 Iteration 8220 | Training loss: 0.4366\n",
            "Epoch 45/100 Iteration 8230 | Training loss: 0.4810\n",
            "Epoch 45/100 Iteration 8240 | Training loss: 0.4680\n",
            "Epoch 45/100 Iteration 8250 | Training loss: 0.4949\n",
            "Epoch 45/100 Iteration 8260 | Training loss: 0.4075\n",
            "Epoch 45/100 Iteration 8270 | Training loss: 0.4144\n",
            "Epoch 45/100 Iteration 8280 | Training loss: 0.4428\n",
            "Epoch 45/100 Iteration 8290 | Training loss: 0.6084\n",
            "Epoch 45/100 Iteration 8300 | Training loss: 0.5793\n",
            "Epoch 45/100 Iteration 8310 | Training loss: 0.4734\n",
            "Epoch 45/100 Iteration 8320 | Training loss: 0.5150\n",
            "Epoch 45/100 Iteration 8330 | Training loss: 0.4660\n",
            "Epoch 45/100 Iteration 8340 | Training loss: 0.4128\n",
            "Epoch 45/100 Iteration 8350 | Training loss: 0.4791\n",
            "Epoch 45/100 Iteration 8360 | Training loss: 0.4020\n",
            "Epoch 45/100 Iteration 8370 | Training loss: 0.4613\n",
            "Epoch 46/100 Iteration 8380 | Training loss: 0.5046\n",
            "Epoch 46/100 Iteration 8390 | Training loss: 0.5137\n",
            "Epoch 46/100 Iteration 8400 | Training loss: 0.4323\n",
            "Epoch 46/100 Iteration 8410 | Training loss: 0.4875\n",
            "Epoch 46/100 Iteration 8420 | Training loss: 0.4770\n",
            "Epoch 46/100 Iteration 8430 | Training loss: 0.4069\n",
            "Epoch 46/100 Iteration 8440 | Training loss: 0.4852\n",
            "Epoch 46/100 Iteration 8450 | Training loss: 0.4787\n",
            "Epoch 46/100 Iteration 8460 | Training loss: 0.4324\n",
            "Epoch 46/100 Iteration 8470 | Training loss: 0.4175\n",
            "Epoch 46/100 Iteration 8480 | Training loss: 0.3925\n",
            "Epoch 46/100 Iteration 8490 | Training loss: 0.4990\n",
            "Epoch 46/100 Iteration 8500 | Training loss: 0.3593\n",
            "Epoch 46/100 Iteration 8510 | Training loss: 0.4300\n",
            "Epoch 46/100 Iteration 8520 | Training loss: 0.4779\n",
            "Epoch 46/100 Iteration 8530 | Training loss: 0.4861\n",
            "Epoch 46/100 Iteration 8540 | Training loss: 0.5678\n",
            "Epoch 46/100 Iteration 8550 | Training loss: 0.3894\n",
            "Epoch 47/100 Iteration 8560 | Training loss: 0.4611\n",
            "Epoch 47/100 Iteration 8570 | Training loss: 0.4350\n",
            "Epoch 47/100 Iteration 8580 | Training loss: 0.5192\n",
            "Epoch 47/100 Iteration 8590 | Training loss: 0.4845\n",
            "Epoch 47/100 Iteration 8600 | Training loss: 0.5848\n",
            "Epoch 47/100 Iteration 8610 | Training loss: 0.4462\n",
            "Epoch 47/100 Iteration 8620 | Training loss: 0.4369\n",
            "Epoch 47/100 Iteration 8630 | Training loss: 0.4493\n",
            "Epoch 47/100 Iteration 8640 | Training loss: 0.4640\n",
            "Epoch 47/100 Iteration 8650 | Training loss: 0.4862\n",
            "Epoch 47/100 Iteration 8660 | Training loss: 0.3727\n",
            "Epoch 47/100 Iteration 8670 | Training loss: 0.4349\n",
            "Epoch 47/100 Iteration 8680 | Training loss: 0.4802\n",
            "Epoch 47/100 Iteration 8690 | Training loss: 0.4207\n",
            "Epoch 47/100 Iteration 8700 | Training loss: 0.5032\n",
            "Epoch 47/100 Iteration 8710 | Training loss: 0.4290\n",
            "Epoch 47/100 Iteration 8720 | Training loss: 0.4763\n",
            "Epoch 47/100 Iteration 8730 | Training loss: 0.5050\n",
            "Epoch 47/100 Iteration 8740 | Training loss: 0.4206\n",
            "Epoch 48/100 Iteration 8750 | Training loss: 0.5107\n",
            "Epoch 48/100 Iteration 8760 | Training loss: 0.3304\n",
            "Epoch 48/100 Iteration 8770 | Training loss: 0.4592\n",
            "Epoch 48/100 Iteration 8780 | Training loss: 0.4740\n",
            "Epoch 48/100 Iteration 8790 | Training loss: 0.4493\n",
            "Epoch 48/100 Iteration 8800 | Training loss: 0.4481\n",
            "Epoch 48/100 Iteration 8810 | Training loss: 0.5231\n",
            "Epoch 48/100 Iteration 8820 | Training loss: 0.4360\n",
            "Epoch 48/100 Iteration 8830 | Training loss: 0.3703\n",
            "Epoch 48/100 Iteration 8840 | Training loss: 0.3893\n",
            "Epoch 48/100 Iteration 8850 | Training loss: 0.4736\n",
            "Epoch 48/100 Iteration 8860 | Training loss: 0.4518\n",
            "Epoch 48/100 Iteration 8870 | Training loss: 0.4671\n",
            "Epoch 48/100 Iteration 8880 | Training loss: 0.4267\n",
            "Epoch 48/100 Iteration 8890 | Training loss: 0.4829\n",
            "Epoch 48/100 Iteration 8900 | Training loss: 0.4428\n",
            "Epoch 48/100 Iteration 8910 | Training loss: 0.5732\n",
            "Epoch 48/100 Iteration 8920 | Training loss: 0.4073\n",
            "Epoch 49/100 Iteration 8930 | Training loss: 0.4069\n",
            "Epoch 49/100 Iteration 8940 | Training loss: 0.4754\n",
            "Epoch 49/100 Iteration 8950 | Training loss: 0.5045\n",
            "Epoch 49/100 Iteration 8960 | Training loss: 0.4796\n",
            "Epoch 49/100 Iteration 8970 | Training loss: 0.5242\n",
            "Epoch 49/100 Iteration 8980 | Training loss: 0.4196\n",
            "Epoch 49/100 Iteration 8990 | Training loss: 0.4779\n",
            "Epoch 49/100 Iteration 9000 | Training loss: 0.4135\n",
            "Epoch 49/100 Iteration 9010 | Training loss: 0.4046\n",
            "Epoch 49/100 Iteration 9020 | Training loss: 0.4533\n",
            "Epoch 49/100 Iteration 9030 | Training loss: 0.3835\n",
            "Epoch 49/100 Iteration 9040 | Training loss: 0.3936\n",
            "Epoch 49/100 Iteration 9050 | Training loss: 0.4530\n",
            "Epoch 49/100 Iteration 9060 | Training loss: 0.4302\n",
            "Epoch 49/100 Iteration 9070 | Training loss: 0.4416\n",
            "Epoch 49/100 Iteration 9080 | Training loss: 0.3900\n",
            "Epoch 49/100 Iteration 9090 | Training loss: 0.4318\n",
            "Epoch 49/100 Iteration 9100 | Training loss: 0.4599\n",
            "Epoch 49/100 Iteration 9110 | Training loss: 0.4381\n",
            "Epoch 50/100 Iteration 9120 | Training loss: 0.4288\n",
            "Epoch 50/100 Iteration 9130 | Training loss: 0.4356\n",
            "Epoch 50/100 Iteration 9140 | Training loss: 0.4993\n",
            "Epoch 50/100 Iteration 9150 | Training loss: 0.4283\n",
            "Epoch 50/100 Iteration 9160 | Training loss: 0.4723\n",
            "Epoch 50/100 Iteration 9170 | Training loss: 0.4591\n",
            "Epoch 50/100 Iteration 9180 | Training loss: 0.4889\n",
            "Epoch 50/100 Iteration 9190 | Training loss: 0.3977\n",
            "Epoch 50/100 Iteration 9200 | Training loss: 0.4021\n",
            "Epoch 50/100 Iteration 9210 | Training loss: 0.4031\n",
            "Epoch 50/100 Iteration 9220 | Training loss: 0.4604\n",
            "Epoch 50/100 Iteration 9230 | Training loss: 0.5068\n",
            "Epoch 50/100 Iteration 9240 | Training loss: 0.4360\n",
            "Epoch 50/100 Iteration 9250 | Training loss: 0.4802\n",
            "Epoch 50/100 Iteration 9260 | Training loss: 0.4422\n",
            "Epoch 50/100 Iteration 9270 | Training loss: 0.3938\n",
            "Epoch 50/100 Iteration 9280 | Training loss: 0.4615\n",
            "Epoch 50/100 Iteration 9290 | Training loss: 0.3807\n",
            "Epoch 50/100 Iteration 9300 | Training loss: 0.4612\n",
            "Epoch 51/100 Iteration 9310 | Training loss: 0.4905\n",
            "Epoch 51/100 Iteration 9320 | Training loss: 0.5036\n",
            "Epoch 51/100 Iteration 9330 | Training loss: 0.4154\n",
            "Epoch 51/100 Iteration 9340 | Training loss: 0.4741\n",
            "Epoch 51/100 Iteration 9350 | Training loss: 0.4703\n",
            "Epoch 51/100 Iteration 9360 | Training loss: 0.4008\n",
            "Epoch 51/100 Iteration 9370 | Training loss: 0.4734\n",
            "Epoch 51/100 Iteration 9380 | Training loss: 0.4701\n",
            "Epoch 51/100 Iteration 9390 | Training loss: 0.4269\n",
            "Epoch 51/100 Iteration 9400 | Training loss: 0.4132\n",
            "Epoch 51/100 Iteration 9410 | Training loss: 0.3882\n",
            "Epoch 51/100 Iteration 9420 | Training loss: 0.4850\n",
            "Epoch 51/100 Iteration 9430 | Training loss: 0.3444\n",
            "Epoch 51/100 Iteration 9440 | Training loss: 0.4213\n",
            "Epoch 51/100 Iteration 9450 | Training loss: 0.4654\n",
            "Epoch 51/100 Iteration 9460 | Training loss: 0.4785\n",
            "Epoch 51/100 Iteration 9470 | Training loss: 0.5660\n",
            "Epoch 51/100 Iteration 9480 | Training loss: 0.3808\n",
            "Epoch 52/100 Iteration 9490 | Training loss: 0.4563\n",
            "Epoch 52/100 Iteration 9500 | Training loss: 0.4287\n",
            "Epoch 52/100 Iteration 9510 | Training loss: 0.5128\n",
            "Epoch 52/100 Iteration 9520 | Training loss: 0.4741\n",
            "Epoch 52/100 Iteration 9530 | Training loss: 0.5674\n",
            "Epoch 52/100 Iteration 9540 | Training loss: 0.4351\n",
            "Epoch 52/100 Iteration 9550 | Training loss: 0.4238\n",
            "Epoch 52/100 Iteration 9560 | Training loss: 0.4362\n",
            "Epoch 52/100 Iteration 9570 | Training loss: 0.4516\n",
            "Epoch 52/100 Iteration 9580 | Training loss: 0.4796\n",
            "Epoch 52/100 Iteration 9590 | Training loss: 0.3666\n",
            "Epoch 52/100 Iteration 9600 | Training loss: 0.4245\n",
            "Epoch 52/100 Iteration 9610 | Training loss: 0.4687\n",
            "Epoch 52/100 Iteration 9620 | Training loss: 0.4162\n",
            "Epoch 52/100 Iteration 9630 | Training loss: 0.4900\n",
            "Epoch 52/100 Iteration 9640 | Training loss: 0.4147\n",
            "Epoch 52/100 Iteration 9650 | Training loss: 0.4665\n",
            "Epoch 52/100 Iteration 9660 | Training loss: 0.4932\n",
            "Epoch 52/100 Iteration 9670 | Training loss: 0.4218\n",
            "Epoch 53/100 Iteration 9680 | Training loss: 0.5004\n",
            "Epoch 53/100 Iteration 9690 | Training loss: 0.3267\n",
            "Epoch 53/100 Iteration 9700 | Training loss: 0.4555\n",
            "Epoch 53/100 Iteration 9710 | Training loss: 0.4681\n",
            "Epoch 53/100 Iteration 9720 | Training loss: 0.4498\n",
            "Epoch 53/100 Iteration 9730 | Training loss: 0.4368\n",
            "Epoch 53/100 Iteration 9740 | Training loss: 0.5128\n",
            "Epoch 53/100 Iteration 9750 | Training loss: 0.4312\n",
            "Epoch 53/100 Iteration 9760 | Training loss: 0.3625\n",
            "Epoch 53/100 Iteration 9770 | Training loss: 0.3845\n",
            "Epoch 53/100 Iteration 9780 | Training loss: 0.4654\n",
            "Epoch 53/100 Iteration 9790 | Training loss: 0.4467\n",
            "Epoch 53/100 Iteration 9800 | Training loss: 0.4531\n",
            "Epoch 53/100 Iteration 9810 | Training loss: 0.4182\n",
            "Epoch 53/100 Iteration 9820 | Training loss: 0.4686\n",
            "Epoch 53/100 Iteration 9830 | Training loss: 0.4306\n",
            "Epoch 53/100 Iteration 9840 | Training loss: 0.5638\n",
            "Epoch 53/100 Iteration 9850 | Training loss: 0.3970\n",
            "Epoch 54/100 Iteration 9860 | Training loss: 0.4036\n",
            "Epoch 54/100 Iteration 9870 | Training loss: 0.4721\n",
            "Epoch 54/100 Iteration 9880 | Training loss: 0.4892\n",
            "Epoch 54/100 Iteration 9890 | Training loss: 0.4653\n",
            "Epoch 54/100 Iteration 9900 | Training loss: 0.5200\n",
            "Epoch 54/100 Iteration 9910 | Training loss: 0.4105\n",
            "Epoch 54/100 Iteration 9920 | Training loss: 0.4744\n",
            "Epoch 54/100 Iteration 9930 | Training loss: 0.4048\n",
            "Epoch 54/100 Iteration 9940 | Training loss: 0.3981\n",
            "Epoch 54/100 Iteration 9950 | Training loss: 0.4438\n",
            "Epoch 54/100 Iteration 9960 | Training loss: 0.3820\n",
            "Epoch 54/100 Iteration 9970 | Training loss: 0.3969\n",
            "Epoch 54/100 Iteration 9980 | Training loss: 0.4487\n",
            "Epoch 54/100 Iteration 9990 | Training loss: 0.4181\n",
            "Epoch 54/100 Iteration 10000 | Training loss: 0.4349\n",
            "Epoch 54/100 Iteration 10010 | Training loss: 0.3820\n",
            "Epoch 54/100 Iteration 10020 | Training loss: 0.4245\n",
            "Epoch 54/100 Iteration 10030 | Training loss: 0.4530\n",
            "Epoch 54/100 Iteration 10040 | Training loss: 0.4286\n",
            "Epoch 55/100 Iteration 10050 | Training loss: 0.4228\n",
            "Epoch 55/100 Iteration 10060 | Training loss: 0.4213\n",
            "Epoch 55/100 Iteration 10070 | Training loss: 0.4869\n",
            "Epoch 55/100 Iteration 10080 | Training loss: 0.4260\n",
            "Epoch 55/100 Iteration 10090 | Training loss: 0.4611\n",
            "Epoch 55/100 Iteration 10100 | Training loss: 0.4564\n",
            "Epoch 55/100 Iteration 10110 | Training loss: 0.4750\n",
            "Epoch 55/100 Iteration 10120 | Training loss: 0.3916\n",
            "Epoch 55/100 Iteration 10130 | Training loss: 0.3947\n",
            "Epoch 55/100 Iteration 10140 | Training loss: 0.3964\n",
            "Epoch 55/100 Iteration 10150 | Training loss: 0.4500\n",
            "Epoch 55/100 Iteration 10160 | Training loss: 0.4948\n",
            "Epoch 55/100 Iteration 10170 | Training loss: 0.4204\n",
            "Epoch 55/100 Iteration 10180 | Training loss: 0.4681\n",
            "Epoch 55/100 Iteration 10190 | Training loss: 0.4313\n",
            "Epoch 55/100 Iteration 10200 | Training loss: 0.3856\n",
            "Epoch 55/100 Iteration 10210 | Training loss: 0.4584\n",
            "Epoch 55/100 Iteration 10220 | Training loss: 0.3783\n",
            "Epoch 55/100 Iteration 10230 | Training loss: 0.4423\n",
            "Epoch 56/100 Iteration 10240 | Training loss: 0.4847\n",
            "Epoch 56/100 Iteration 10250 | Training loss: 0.4877\n",
            "Epoch 56/100 Iteration 10260 | Training loss: 0.4066\n",
            "Epoch 56/100 Iteration 10270 | Training loss: 0.4700\n",
            "Epoch 56/100 Iteration 10280 | Training loss: 0.4577\n",
            "Epoch 56/100 Iteration 10290 | Training loss: 0.3774\n",
            "Epoch 56/100 Iteration 10300 | Training loss: 0.4632\n",
            "Epoch 56/100 Iteration 10310 | Training loss: 0.4608\n",
            "Epoch 56/100 Iteration 10320 | Training loss: 0.4191\n",
            "Epoch 56/100 Iteration 10330 | Training loss: 0.4022\n",
            "Epoch 56/100 Iteration 10340 | Training loss: 0.3767\n",
            "Epoch 56/100 Iteration 10350 | Training loss: 0.4860\n",
            "Epoch 56/100 Iteration 10360 | Training loss: 0.3347\n",
            "Epoch 56/100 Iteration 10370 | Training loss: 0.4143\n",
            "Epoch 56/100 Iteration 10380 | Training loss: 0.4601\n",
            "Epoch 56/100 Iteration 10390 | Training loss: 0.4697\n",
            "Epoch 56/100 Iteration 10400 | Training loss: 0.5526\n",
            "Epoch 56/100 Iteration 10410 | Training loss: 0.3803\n",
            "Epoch 57/100 Iteration 10420 | Training loss: 0.4444\n",
            "Epoch 57/100 Iteration 10430 | Training loss: 0.4177\n",
            "Epoch 57/100 Iteration 10440 | Training loss: 0.5107\n",
            "Epoch 57/100 Iteration 10450 | Training loss: 0.4631\n",
            "Epoch 57/100 Iteration 10460 | Training loss: 0.5705\n",
            "Epoch 57/100 Iteration 10470 | Training loss: 0.4337\n",
            "Epoch 57/100 Iteration 10480 | Training loss: 0.4194\n",
            "Epoch 57/100 Iteration 10490 | Training loss: 0.4418\n",
            "Epoch 57/100 Iteration 10500 | Training loss: 0.4466\n",
            "Epoch 57/100 Iteration 10510 | Training loss: 0.4756\n",
            "Epoch 57/100 Iteration 10520 | Training loss: 0.3647\n",
            "Epoch 57/100 Iteration 10530 | Training loss: 0.4201\n",
            "Epoch 57/100 Iteration 10540 | Training loss: 0.4624\n",
            "Epoch 57/100 Iteration 10550 | Training loss: 0.4097\n",
            "Epoch 57/100 Iteration 10560 | Training loss: 0.4808\n",
            "Epoch 57/100 Iteration 10570 | Training loss: 0.4111\n",
            "Epoch 57/100 Iteration 10580 | Training loss: 0.4645\n",
            "Epoch 57/100 Iteration 10590 | Training loss: 0.4883\n",
            "Epoch 57/100 Iteration 10600 | Training loss: 0.4123\n",
            "Epoch 58/100 Iteration 10610 | Training loss: 0.4987\n",
            "Epoch 58/100 Iteration 10620 | Training loss: 0.3209\n",
            "Epoch 58/100 Iteration 10630 | Training loss: 0.4453\n",
            "Epoch 58/100 Iteration 10640 | Training loss: 0.4591\n",
            "Epoch 58/100 Iteration 10650 | Training loss: 0.4357\n",
            "Epoch 58/100 Iteration 10660 | Training loss: 0.4368\n",
            "Epoch 58/100 Iteration 10670 | Training loss: 0.5011\n",
            "Epoch 58/100 Iteration 10680 | Training loss: 0.4290\n",
            "Epoch 58/100 Iteration 10690 | Training loss: 0.3614\n",
            "Epoch 58/100 Iteration 10700 | Training loss: 0.3835\n",
            "Epoch 58/100 Iteration 10710 | Training loss: 0.4533\n",
            "Epoch 58/100 Iteration 10720 | Training loss: 0.4372\n",
            "Epoch 58/100 Iteration 10730 | Training loss: 0.4448\n",
            "Epoch 58/100 Iteration 10740 | Training loss: 0.4100\n",
            "Epoch 58/100 Iteration 10750 | Training loss: 0.4648\n",
            "Epoch 58/100 Iteration 10760 | Training loss: 0.4263\n",
            "Epoch 58/100 Iteration 10770 | Training loss: 0.5560\n",
            "Epoch 58/100 Iteration 10780 | Training loss: 0.3918\n",
            "Epoch 59/100 Iteration 10790 | Training loss: 0.4011\n",
            "Epoch 59/100 Iteration 10800 | Training loss: 0.4711\n",
            "Epoch 59/100 Iteration 10810 | Training loss: 0.4897\n",
            "Epoch 59/100 Iteration 10820 | Training loss: 0.4597\n",
            "Epoch 59/100 Iteration 10830 | Training loss: 0.5155\n",
            "Epoch 59/100 Iteration 10840 | Training loss: 0.4072\n",
            "Epoch 59/100 Iteration 10850 | Training loss: 0.4659\n",
            "Epoch 59/100 Iteration 10860 | Training loss: 0.4012\n",
            "Epoch 59/100 Iteration 10870 | Training loss: 0.3874\n",
            "Epoch 59/100 Iteration 10880 | Training loss: 0.4365\n",
            "Epoch 59/100 Iteration 10890 | Training loss: 0.3717\n",
            "Epoch 59/100 Iteration 10900 | Training loss: 0.3914\n",
            "Epoch 59/100 Iteration 10910 | Training loss: 0.4456\n",
            "Epoch 59/100 Iteration 10920 | Training loss: 0.4157\n",
            "Epoch 59/100 Iteration 10930 | Training loss: 0.4284\n",
            "Epoch 59/100 Iteration 10940 | Training loss: 0.3805\n",
            "Epoch 59/100 Iteration 10950 | Training loss: 0.4198\n",
            "Epoch 59/100 Iteration 10960 | Training loss: 0.4501\n",
            "Epoch 59/100 Iteration 10970 | Training loss: 0.4239\n",
            "Epoch 60/100 Iteration 10980 | Training loss: 0.4188\n",
            "Epoch 60/100 Iteration 10990 | Training loss: 0.4175\n",
            "Epoch 60/100 Iteration 11000 | Training loss: 0.4785\n",
            "Epoch 60/100 Iteration 11010 | Training loss: 0.4260\n",
            "Epoch 60/100 Iteration 11020 | Training loss: 0.4614\n",
            "Epoch 60/100 Iteration 11030 | Training loss: 0.4479\n",
            "Epoch 60/100 Iteration 11040 | Training loss: 0.4702\n",
            "Epoch 60/100 Iteration 11050 | Training loss: 0.3832\n",
            "Epoch 60/100 Iteration 11060 | Training loss: 0.3905\n",
            "Epoch 60/100 Iteration 11070 | Training loss: 0.3911\n",
            "Epoch 60/100 Iteration 11080 | Training loss: 0.4299\n",
            "Epoch 60/100 Iteration 11090 | Training loss: 0.4883\n",
            "Epoch 60/100 Iteration 11100 | Training loss: 0.4103\n",
            "Epoch 60/100 Iteration 11110 | Training loss: 0.4702\n",
            "Epoch 60/100 Iteration 11120 | Training loss: 0.4294\n",
            "Epoch 60/100 Iteration 11130 | Training loss: 0.3815\n",
            "Epoch 60/100 Iteration 11140 | Training loss: 0.4555\n",
            "Epoch 60/100 Iteration 11150 | Training loss: 0.3714\n",
            "Epoch 60/100 Iteration 11160 | Training loss: 0.4381\n",
            "Epoch 61/100 Iteration 11170 | Training loss: 0.4697\n",
            "Epoch 61/100 Iteration 11180 | Training loss: 0.4859\n",
            "Epoch 61/100 Iteration 11190 | Training loss: 0.4052\n",
            "Epoch 61/100 Iteration 11200 | Training loss: 0.4646\n",
            "Epoch 61/100 Iteration 11210 | Training loss: 0.4533\n",
            "Epoch 61/100 Iteration 11220 | Training loss: 0.3812\n",
            "Epoch 61/100 Iteration 11230 | Training loss: 0.4554\n",
            "Epoch 61/100 Iteration 11240 | Training loss: 0.4582\n",
            "Epoch 61/100 Iteration 11250 | Training loss: 0.4114\n",
            "Epoch 61/100 Iteration 11260 | Training loss: 0.3950\n",
            "Epoch 61/100 Iteration 11270 | Training loss: 0.3710\n",
            "Epoch 61/100 Iteration 11280 | Training loss: 0.4824\n",
            "Epoch 61/100 Iteration 11290 | Training loss: 0.3398\n",
            "Epoch 61/100 Iteration 11300 | Training loss: 0.4110\n",
            "Epoch 61/100 Iteration 11310 | Training loss: 0.4512\n",
            "Epoch 61/100 Iteration 11320 | Training loss: 0.4598\n",
            "Epoch 61/100 Iteration 11330 | Training loss: 0.5437\n",
            "Epoch 61/100 Iteration 11340 | Training loss: 0.3726\n",
            "Epoch 62/100 Iteration 11350 | Training loss: 0.4383\n",
            "Epoch 62/100 Iteration 11360 | Training loss: 0.4098\n",
            "Epoch 62/100 Iteration 11370 | Training loss: 0.5009\n",
            "Epoch 62/100 Iteration 11380 | Training loss: 0.4626\n",
            "Epoch 62/100 Iteration 11390 | Training loss: 0.5578\n",
            "Epoch 62/100 Iteration 11400 | Training loss: 0.4204\n",
            "Epoch 62/100 Iteration 11410 | Training loss: 0.4056\n",
            "Epoch 62/100 Iteration 11420 | Training loss: 0.4287\n",
            "Epoch 62/100 Iteration 11430 | Training loss: 0.4387\n",
            "Epoch 62/100 Iteration 11440 | Training loss: 0.4690\n",
            "Epoch 62/100 Iteration 11450 | Training loss: 0.3536\n",
            "Epoch 62/100 Iteration 11460 | Training loss: 0.4132\n",
            "Epoch 62/100 Iteration 11470 | Training loss: 0.4571\n",
            "Epoch 62/100 Iteration 11480 | Training loss: 0.3989\n",
            "Epoch 62/100 Iteration 11490 | Training loss: 0.4739\n",
            "Epoch 62/100 Iteration 11500 | Training loss: 0.4026\n",
            "Epoch 62/100 Iteration 11510 | Training loss: 0.4556\n",
            "Epoch 62/100 Iteration 11520 | Training loss: 0.4785\n",
            "Epoch 62/100 Iteration 11530 | Training loss: 0.4077\n",
            "Epoch 63/100 Iteration 11540 | Training loss: 0.4959\n",
            "Epoch 63/100 Iteration 11550 | Training loss: 0.3113\n",
            "Epoch 63/100 Iteration 11560 | Training loss: 0.4410\n",
            "Epoch 63/100 Iteration 11570 | Training loss: 0.4488\n",
            "Epoch 63/100 Iteration 11580 | Training loss: 0.4284\n",
            "Epoch 63/100 Iteration 11590 | Training loss: 0.4258\n",
            "Epoch 63/100 Iteration 11600 | Training loss: 0.4896\n",
            "Epoch 63/100 Iteration 11610 | Training loss: 0.4253\n",
            "Epoch 63/100 Iteration 11620 | Training loss: 0.3530\n",
            "Epoch 63/100 Iteration 11630 | Training loss: 0.3711\n",
            "Epoch 63/100 Iteration 11640 | Training loss: 0.4501\n",
            "Epoch 63/100 Iteration 11650 | Training loss: 0.4315\n",
            "Epoch 63/100 Iteration 11660 | Training loss: 0.4452\n",
            "Epoch 63/100 Iteration 11670 | Training loss: 0.4032\n",
            "Epoch 63/100 Iteration 11680 | Training loss: 0.4581\n",
            "Epoch 63/100 Iteration 11690 | Training loss: 0.4157\n",
            "Epoch 63/100 Iteration 11700 | Training loss: 0.5403\n",
            "Epoch 63/100 Iteration 11710 | Training loss: 0.3792\n",
            "Epoch 64/100 Iteration 11720 | Training loss: 0.3929\n",
            "Epoch 64/100 Iteration 11730 | Training loss: 0.4667\n",
            "Epoch 64/100 Iteration 11740 | Training loss: 0.4770\n",
            "Epoch 64/100 Iteration 11750 | Training loss: 0.4528\n",
            "Epoch 64/100 Iteration 11760 | Training loss: 0.5075\n",
            "Epoch 64/100 Iteration 11770 | Training loss: 0.4026\n",
            "Epoch 64/100 Iteration 11780 | Training loss: 0.4532\n",
            "Epoch 64/100 Iteration 11790 | Training loss: 0.3923\n",
            "Epoch 64/100 Iteration 11800 | Training loss: 0.3886\n",
            "Epoch 64/100 Iteration 11810 | Training loss: 0.4337\n",
            "Epoch 64/100 Iteration 11820 | Training loss: 0.3678\n",
            "Epoch 64/100 Iteration 11830 | Training loss: 0.3768\n",
            "Epoch 64/100 Iteration 11840 | Training loss: 0.4335\n",
            "Epoch 64/100 Iteration 11850 | Training loss: 0.4063\n",
            "Epoch 64/100 Iteration 11860 | Training loss: 0.4274\n",
            "Epoch 64/100 Iteration 11870 | Training loss: 0.3677\n",
            "Epoch 64/100 Iteration 11880 | Training loss: 0.4115\n",
            "Epoch 64/100 Iteration 11890 | Training loss: 0.4455\n",
            "Epoch 64/100 Iteration 11900 | Training loss: 0.4248\n",
            "Epoch 65/100 Iteration 11910 | Training loss: 0.4130\n",
            "Epoch 65/100 Iteration 11920 | Training loss: 0.4103\n",
            "Epoch 65/100 Iteration 11930 | Training loss: 0.4707\n",
            "Epoch 65/100 Iteration 11940 | Training loss: 0.4085\n",
            "Epoch 65/100 Iteration 11950 | Training loss: 0.4514\n",
            "Epoch 65/100 Iteration 11960 | Training loss: 0.4344\n",
            "Epoch 65/100 Iteration 11970 | Training loss: 0.4572\n",
            "Epoch 65/100 Iteration 11980 | Training loss: 0.3778\n",
            "Epoch 65/100 Iteration 11990 | Training loss: 0.3793\n",
            "Epoch 65/100 Iteration 12000 | Training loss: 0.3870\n",
            "Epoch 65/100 Iteration 12010 | Training loss: 0.4315\n",
            "Epoch 65/100 Iteration 12020 | Training loss: 0.4777\n",
            "Epoch 65/100 Iteration 12030 | Training loss: 0.4098\n",
            "Epoch 65/100 Iteration 12040 | Training loss: 0.4682\n",
            "Epoch 65/100 Iteration 12050 | Training loss: 0.4265\n",
            "Epoch 65/100 Iteration 12060 | Training loss: 0.3793\n",
            "Epoch 65/100 Iteration 12070 | Training loss: 0.4475\n",
            "Epoch 65/100 Iteration 12080 | Training loss: 0.3716\n",
            "Epoch 65/100 Iteration 12090 | Training loss: 0.4309\n",
            "Epoch 66/100 Iteration 12100 | Training loss: 0.4680\n",
            "Epoch 66/100 Iteration 12110 | Training loss: 0.4828\n",
            "Epoch 66/100 Iteration 12120 | Training loss: 0.4045\n",
            "Epoch 66/100 Iteration 12130 | Training loss: 0.4536\n",
            "Epoch 66/100 Iteration 12140 | Training loss: 0.4499\n",
            "Epoch 66/100 Iteration 12150 | Training loss: 0.3735\n",
            "Epoch 66/100 Iteration 12160 | Training loss: 0.4478\n",
            "Epoch 66/100 Iteration 12170 | Training loss: 0.4508\n",
            "Epoch 66/100 Iteration 12180 | Training loss: 0.4032\n",
            "Epoch 66/100 Iteration 12190 | Training loss: 0.3885\n",
            "Epoch 66/100 Iteration 12200 | Training loss: 0.3696\n",
            "Epoch 66/100 Iteration 12210 | Training loss: 0.4700\n",
            "Epoch 66/100 Iteration 12220 | Training loss: 0.3294\n",
            "Epoch 66/100 Iteration 12230 | Training loss: 0.3987\n",
            "Epoch 66/100 Iteration 12240 | Training loss: 0.4429\n",
            "Epoch 66/100 Iteration 12250 | Training loss: 0.4545\n",
            "Epoch 66/100 Iteration 12260 | Training loss: 0.5474\n",
            "Epoch 66/100 Iteration 12270 | Training loss: 0.3690\n",
            "Epoch 67/100 Iteration 12280 | Training loss: 0.4371\n",
            "Epoch 67/100 Iteration 12290 | Training loss: 0.4037\n",
            "Epoch 67/100 Iteration 12300 | Training loss: 0.4999\n",
            "Epoch 67/100 Iteration 12310 | Training loss: 0.4548\n",
            "Epoch 67/100 Iteration 12320 | Training loss: 0.5507\n",
            "Epoch 67/100 Iteration 12330 | Training loss: 0.4181\n",
            "Epoch 67/100 Iteration 12340 | Training loss: 0.4042\n",
            "Epoch 67/100 Iteration 12350 | Training loss: 0.4206\n",
            "Epoch 67/100 Iteration 12360 | Training loss: 0.4366\n",
            "Epoch 67/100 Iteration 12370 | Training loss: 0.4602\n",
            "Epoch 67/100 Iteration 12380 | Training loss: 0.3487\n",
            "Epoch 67/100 Iteration 12390 | Training loss: 0.4111\n",
            "Epoch 67/100 Iteration 12400 | Training loss: 0.4421\n",
            "Epoch 67/100 Iteration 12410 | Training loss: 0.3968\n",
            "Epoch 67/100 Iteration 12420 | Training loss: 0.4667\n",
            "Epoch 67/100 Iteration 12430 | Training loss: 0.4019\n",
            "Epoch 67/100 Iteration 12440 | Training loss: 0.4431\n",
            "Epoch 67/100 Iteration 12450 | Training loss: 0.4710\n",
            "Epoch 67/100 Iteration 12460 | Training loss: 0.4013\n",
            "Epoch 68/100 Iteration 12470 | Training loss: 0.4864\n",
            "Epoch 68/100 Iteration 12480 | Training loss: 0.3076\n",
            "Epoch 68/100 Iteration 12490 | Training loss: 0.4348\n",
            "Epoch 68/100 Iteration 12500 | Training loss: 0.4488\n",
            "Epoch 68/100 Iteration 12510 | Training loss: 0.4247\n",
            "Epoch 68/100 Iteration 12520 | Training loss: 0.4204\n",
            "Epoch 68/100 Iteration 12530 | Training loss: 0.4889\n",
            "Epoch 68/100 Iteration 12540 | Training loss: 0.4162\n",
            "Epoch 68/100 Iteration 12550 | Training loss: 0.3465\n",
            "Epoch 68/100 Iteration 12560 | Training loss: 0.3727\n",
            "Epoch 68/100 Iteration 12570 | Training loss: 0.4408\n",
            "Epoch 68/100 Iteration 12580 | Training loss: 0.4241\n",
            "Epoch 68/100 Iteration 12590 | Training loss: 0.4348\n",
            "Epoch 68/100 Iteration 12600 | Training loss: 0.4141\n",
            "Epoch 68/100 Iteration 12610 | Training loss: 0.4564\n",
            "Epoch 68/100 Iteration 12620 | Training loss: 0.4212\n",
            "Epoch 68/100 Iteration 12630 | Training loss: 0.5362\n",
            "Epoch 68/100 Iteration 12640 | Training loss: 0.3848\n",
            "Epoch 69/100 Iteration 12650 | Training loss: 0.3905\n",
            "Epoch 69/100 Iteration 12660 | Training loss: 0.4558\n",
            "Epoch 69/100 Iteration 12670 | Training loss: 0.4706\n",
            "Epoch 69/100 Iteration 12680 | Training loss: 0.4489\n",
            "Epoch 69/100 Iteration 12690 | Training loss: 0.5012\n",
            "Epoch 69/100 Iteration 12700 | Training loss: 0.3910\n",
            "Epoch 69/100 Iteration 12710 | Training loss: 0.4528\n",
            "Epoch 69/100 Iteration 12720 | Training loss: 0.3824\n",
            "Epoch 69/100 Iteration 12730 | Training loss: 0.3793\n",
            "Epoch 69/100 Iteration 12740 | Training loss: 0.4307\n",
            "Epoch 69/100 Iteration 12750 | Training loss: 0.3657\n",
            "Epoch 69/100 Iteration 12760 | Training loss: 0.3709\n",
            "Epoch 69/100 Iteration 12770 | Training loss: 0.4276\n",
            "Epoch 69/100 Iteration 12780 | Training loss: 0.4035\n",
            "Epoch 69/100 Iteration 12790 | Training loss: 0.4216\n",
            "Epoch 69/100 Iteration 12800 | Training loss: 0.3670\n",
            "Epoch 69/100 Iteration 12810 | Training loss: 0.4144\n",
            "Epoch 69/100 Iteration 12820 | Training loss: 0.4454\n",
            "Epoch 69/100 Iteration 12830 | Training loss: 0.4211\n",
            "Epoch 70/100 Iteration 12840 | Training loss: 0.4070\n",
            "Epoch 70/100 Iteration 12850 | Training loss: 0.4069\n",
            "Epoch 70/100 Iteration 12860 | Training loss: 0.4627\n",
            "Epoch 70/100 Iteration 12870 | Training loss: 0.4083\n",
            "Epoch 70/100 Iteration 12880 | Training loss: 0.4458\n",
            "Epoch 70/100 Iteration 12890 | Training loss: 0.4382\n",
            "Epoch 70/100 Iteration 12900 | Training loss: 0.4485\n",
            "Epoch 70/100 Iteration 12910 | Training loss: 0.3639\n",
            "Epoch 70/100 Iteration 12920 | Training loss: 0.3806\n",
            "Epoch 70/100 Iteration 12930 | Training loss: 0.3823\n",
            "Epoch 70/100 Iteration 12940 | Training loss: 0.4253\n",
            "Epoch 70/100 Iteration 12950 | Training loss: 0.4752\n",
            "Epoch 70/100 Iteration 12960 | Training loss: 0.4038\n",
            "Epoch 70/100 Iteration 12970 | Training loss: 0.4520\n",
            "Epoch 70/100 Iteration 12980 | Training loss: 0.4249\n",
            "Epoch 70/100 Iteration 12990 | Training loss: 0.3704\n",
            "Epoch 70/100 Iteration 13000 | Training loss: 0.4437\n",
            "Epoch 70/100 Iteration 13010 | Training loss: 0.3627\n",
            "Epoch 70/100 Iteration 13020 | Training loss: 0.4268\n",
            "Epoch 71/100 Iteration 13030 | Training loss: 0.4555\n",
            "Epoch 71/100 Iteration 13040 | Training loss: 0.4728\n",
            "Epoch 71/100 Iteration 13050 | Training loss: 0.3923\n",
            "Epoch 71/100 Iteration 13060 | Training loss: 0.4496\n",
            "Epoch 71/100 Iteration 13070 | Training loss: 0.4442\n",
            "Epoch 71/100 Iteration 13080 | Training loss: 0.3640\n",
            "Epoch 71/100 Iteration 13090 | Training loss: 0.4409\n",
            "Epoch 71/100 Iteration 13100 | Training loss: 0.4406\n",
            "Epoch 71/100 Iteration 13110 | Training loss: 0.3983\n",
            "Epoch 71/100 Iteration 13120 | Training loss: 0.3901\n",
            "Epoch 71/100 Iteration 13130 | Training loss: 0.3632\n",
            "Epoch 71/100 Iteration 13140 | Training loss: 0.4645\n",
            "Epoch 71/100 Iteration 13150 | Training loss: 0.3187\n",
            "Epoch 71/100 Iteration 13160 | Training loss: 0.3973\n",
            "Epoch 71/100 Iteration 13170 | Training loss: 0.4352\n",
            "Epoch 71/100 Iteration 13180 | Training loss: 0.4472\n",
            "Epoch 71/100 Iteration 13190 | Training loss: 0.5383\n",
            "Epoch 71/100 Iteration 13200 | Training loss: 0.3654\n",
            "Epoch 72/100 Iteration 13210 | Training loss: 0.4387\n",
            "Epoch 72/100 Iteration 13220 | Training loss: 0.4042\n",
            "Epoch 72/100 Iteration 13230 | Training loss: 0.4945\n",
            "Epoch 72/100 Iteration 13240 | Training loss: 0.4556\n",
            "Epoch 72/100 Iteration 13250 | Training loss: 0.5504\n",
            "Epoch 72/100 Iteration 13260 | Training loss: 0.4123\n",
            "Epoch 72/100 Iteration 13270 | Training loss: 0.4014\n",
            "Epoch 72/100 Iteration 13280 | Training loss: 0.4137\n",
            "Epoch 72/100 Iteration 13290 | Training loss: 0.4243\n",
            "Epoch 72/100 Iteration 13300 | Training loss: 0.4569\n",
            "Epoch 72/100 Iteration 13310 | Training loss: 0.3434\n",
            "Epoch 72/100 Iteration 13320 | Training loss: 0.4031\n",
            "Epoch 72/100 Iteration 13330 | Training loss: 0.4386\n",
            "Epoch 72/100 Iteration 13340 | Training loss: 0.3976\n",
            "Epoch 72/100 Iteration 13350 | Training loss: 0.4641\n",
            "Epoch 72/100 Iteration 13360 | Training loss: 0.3947\n",
            "Epoch 72/100 Iteration 13370 | Training loss: 0.4525\n",
            "Epoch 72/100 Iteration 13380 | Training loss: 0.4697\n",
            "Epoch 72/100 Iteration 13390 | Training loss: 0.3988\n",
            "Epoch 73/100 Iteration 13400 | Training loss: 0.4823\n",
            "Epoch 73/100 Iteration 13410 | Training loss: 0.3064\n",
            "Epoch 73/100 Iteration 13420 | Training loss: 0.4263\n",
            "Epoch 73/100 Iteration 13430 | Training loss: 0.4426\n",
            "Epoch 73/100 Iteration 13440 | Training loss: 0.4220\n",
            "Epoch 73/100 Iteration 13450 | Training loss: 0.4119\n",
            "Epoch 73/100 Iteration 13460 | Training loss: 0.4812\n",
            "Epoch 73/100 Iteration 13470 | Training loss: 0.4100\n",
            "Epoch 73/100 Iteration 13480 | Training loss: 0.3459\n",
            "Epoch 73/100 Iteration 13490 | Training loss: 0.3655\n",
            "Epoch 73/100 Iteration 13500 | Training loss: 0.4469\n",
            "Epoch 73/100 Iteration 13510 | Training loss: 0.4329\n",
            "Epoch 73/100 Iteration 13520 | Training loss: 0.4504\n",
            "Epoch 73/100 Iteration 13530 | Training loss: 0.4027\n",
            "Epoch 73/100 Iteration 13540 | Training loss: 0.4481\n",
            "Epoch 73/100 Iteration 13550 | Training loss: 0.4084\n",
            "Epoch 73/100 Iteration 13560 | Training loss: 0.5305\n",
            "Epoch 73/100 Iteration 13570 | Training loss: 0.3751\n",
            "Epoch 74/100 Iteration 13580 | Training loss: 0.3902\n",
            "Epoch 74/100 Iteration 13590 | Training loss: 0.4808\n",
            "Epoch 74/100 Iteration 13600 | Training loss: 0.4928\n",
            "Epoch 74/100 Iteration 13610 | Training loss: 0.4615\n",
            "Epoch 74/100 Iteration 13620 | Training loss: 0.5038\n",
            "Epoch 74/100 Iteration 13630 | Training loss: 0.4089\n",
            "Epoch 74/100 Iteration 13640 | Training loss: 0.4592\n",
            "Epoch 74/100 Iteration 13650 | Training loss: 0.3808\n",
            "Epoch 74/100 Iteration 13660 | Training loss: 0.3790\n",
            "Epoch 74/100 Iteration 13670 | Training loss: 0.4296\n",
            "Epoch 74/100 Iteration 13680 | Training loss: 0.3609\n",
            "Epoch 74/100 Iteration 13690 | Training loss: 0.3813\n",
            "Epoch 74/100 Iteration 13700 | Training loss: 0.4345\n",
            "Epoch 74/100 Iteration 13710 | Training loss: 0.3989\n",
            "Epoch 74/100 Iteration 13720 | Training loss: 0.4232\n",
            "Epoch 74/100 Iteration 13730 | Training loss: 0.3610\n",
            "Epoch 74/100 Iteration 13740 | Training loss: 0.4150\n",
            "Epoch 74/100 Iteration 13750 | Training loss: 0.4286\n",
            "Epoch 74/100 Iteration 13760 | Training loss: 0.4007\n",
            "Epoch 75/100 Iteration 13770 | Training loss: 0.4032\n",
            "Epoch 75/100 Iteration 13780 | Training loss: 0.4137\n",
            "Epoch 75/100 Iteration 13790 | Training loss: 0.4685\n",
            "Epoch 75/100 Iteration 13800 | Training loss: 0.4039\n",
            "Epoch 75/100 Iteration 13810 | Training loss: 0.4449\n",
            "Epoch 75/100 Iteration 13820 | Training loss: 0.4335\n",
            "Epoch 75/100 Iteration 13830 | Training loss: 0.4562\n",
            "Epoch 75/100 Iteration 13840 | Training loss: 0.3681\n",
            "Epoch 75/100 Iteration 13850 | Training loss: 0.3815\n",
            "Epoch 75/100 Iteration 13860 | Training loss: 0.3809\n",
            "Epoch 75/100 Iteration 13870 | Training loss: 0.4202\n",
            "Epoch 75/100 Iteration 13880 | Training loss: 0.4660\n",
            "Epoch 75/100 Iteration 13890 | Training loss: 0.3966\n",
            "Epoch 75/100 Iteration 13900 | Training loss: 0.4503\n",
            "Epoch 75/100 Iteration 13910 | Training loss: 0.4234\n",
            "Epoch 75/100 Iteration 13920 | Training loss: 0.3685\n",
            "Epoch 75/100 Iteration 13930 | Training loss: 0.4444\n",
            "Epoch 75/100 Iteration 13940 | Training loss: 0.3531\n",
            "Epoch 75/100 Iteration 13950 | Training loss: 0.4143\n",
            "Epoch 76/100 Iteration 13960 | Training loss: 0.4532\n",
            "Epoch 76/100 Iteration 13970 | Training loss: 0.4634\n",
            "Epoch 76/100 Iteration 13980 | Training loss: 0.3882\n",
            "Epoch 76/100 Iteration 13990 | Training loss: 0.4443\n",
            "Epoch 76/100 Iteration 14000 | Training loss: 0.4345\n",
            "Epoch 76/100 Iteration 14010 | Training loss: 0.3666\n",
            "Epoch 76/100 Iteration 14020 | Training loss: 0.4365\n",
            "Epoch 76/100 Iteration 14030 | Training loss: 0.4359\n",
            "Epoch 76/100 Iteration 14040 | Training loss: 0.4007\n",
            "Epoch 76/100 Iteration 14050 | Training loss: 0.3760\n",
            "Epoch 76/100 Iteration 14060 | Training loss: 0.3521\n",
            "Epoch 76/100 Iteration 14070 | Training loss: 0.4562\n",
            "Epoch 76/100 Iteration 14080 | Training loss: 0.3136\n",
            "Epoch 76/100 Iteration 14090 | Training loss: 0.3952\n",
            "Epoch 76/100 Iteration 14100 | Training loss: 0.4327\n",
            "Epoch 76/100 Iteration 14110 | Training loss: 0.4455\n",
            "Epoch 76/100 Iteration 14120 | Training loss: 0.5315\n",
            "Epoch 76/100 Iteration 14130 | Training loss: 0.3578\n",
            "Epoch 77/100 Iteration 14140 | Training loss: 0.4323\n",
            "Epoch 77/100 Iteration 14150 | Training loss: 0.3945\n",
            "Epoch 77/100 Iteration 14160 | Training loss: 0.4868\n",
            "Epoch 77/100 Iteration 14170 | Training loss: 0.4413\n",
            "Epoch 77/100 Iteration 14180 | Training loss: 0.5458\n",
            "Epoch 77/100 Iteration 14190 | Training loss: 0.4140\n",
            "Epoch 77/100 Iteration 14200 | Training loss: 0.3924\n",
            "Epoch 77/100 Iteration 14210 | Training loss: 0.4014\n",
            "Epoch 77/100 Iteration 14220 | Training loss: 0.4225\n",
            "Epoch 77/100 Iteration 14230 | Training loss: 0.4485\n",
            "Epoch 77/100 Iteration 14240 | Training loss: 0.3459\n",
            "Epoch 77/100 Iteration 14250 | Training loss: 0.3923\n",
            "Epoch 77/100 Iteration 14260 | Training loss: 0.4346\n",
            "Epoch 77/100 Iteration 14270 | Training loss: 0.3916\n",
            "Epoch 77/100 Iteration 14280 | Training loss: 0.4623\n",
            "Epoch 77/100 Iteration 14290 | Training loss: 0.3923\n",
            "Epoch 77/100 Iteration 14300 | Training loss: 0.4422\n",
            "Epoch 77/100 Iteration 14310 | Training loss: 0.4602\n",
            "Epoch 77/100 Iteration 14320 | Training loss: 0.3967\n",
            "Epoch 78/100 Iteration 14330 | Training loss: 0.4793\n",
            "Epoch 78/100 Iteration 14340 | Training loss: 0.3034\n",
            "Epoch 78/100 Iteration 14350 | Training loss: 0.4300\n",
            "Epoch 78/100 Iteration 14360 | Training loss: 0.4372\n",
            "Epoch 78/100 Iteration 14370 | Training loss: 0.4140\n",
            "Epoch 78/100 Iteration 14380 | Training loss: 0.4177\n",
            "Epoch 78/100 Iteration 14390 | Training loss: 0.4783\n",
            "Epoch 78/100 Iteration 14400 | Training loss: 0.4117\n",
            "Epoch 78/100 Iteration 14410 | Training loss: 0.3422\n",
            "Epoch 78/100 Iteration 14420 | Training loss: 0.3632\n",
            "Epoch 78/100 Iteration 14430 | Training loss: 0.4298\n",
            "Epoch 78/100 Iteration 14440 | Training loss: 0.4147\n",
            "Epoch 78/100 Iteration 14450 | Training loss: 0.4348\n",
            "Epoch 78/100 Iteration 14460 | Training loss: 0.3938\n",
            "Epoch 78/100 Iteration 14470 | Training loss: 0.4515\n",
            "Epoch 78/100 Iteration 14480 | Training loss: 0.4073\n",
            "Epoch 78/100 Iteration 14490 | Training loss: 0.5297\n",
            "Epoch 78/100 Iteration 14500 | Training loss: 0.3695\n",
            "Epoch 79/100 Iteration 14510 | Training loss: 0.3841\n",
            "Epoch 79/100 Iteration 14520 | Training loss: 0.4541\n",
            "Epoch 79/100 Iteration 14530 | Training loss: 0.4607\n",
            "Epoch 79/100 Iteration 14540 | Training loss: 0.4390\n",
            "Epoch 79/100 Iteration 14550 | Training loss: 0.4931\n",
            "Epoch 79/100 Iteration 14560 | Training loss: 0.3825\n",
            "Epoch 79/100 Iteration 14570 | Training loss: 0.4461\n",
            "Epoch 79/100 Iteration 14580 | Training loss: 0.3709\n",
            "Epoch 79/100 Iteration 14590 | Training loss: 0.3709\n",
            "Epoch 79/100 Iteration 14600 | Training loss: 0.4217\n",
            "Epoch 79/100 Iteration 14610 | Training loss: 0.3530\n",
            "Epoch 79/100 Iteration 14620 | Training loss: 0.3632\n",
            "Epoch 79/100 Iteration 14630 | Training loss: 0.4164\n",
            "Epoch 79/100 Iteration 14640 | Training loss: 0.3992\n",
            "Epoch 79/100 Iteration 14650 | Training loss: 0.4129\n",
            "Epoch 79/100 Iteration 14660 | Training loss: 0.3531\n",
            "Epoch 79/100 Iteration 14670 | Training loss: 0.4029\n",
            "Epoch 79/100 Iteration 14680 | Training loss: 0.4288\n",
            "Epoch 79/100 Iteration 14690 | Training loss: 0.3975\n",
            "Epoch 80/100 Iteration 14700 | Training loss: 0.3970\n",
            "Epoch 80/100 Iteration 14710 | Training loss: 0.4051\n",
            "Epoch 80/100 Iteration 14720 | Training loss: 0.4518\n",
            "Epoch 80/100 Iteration 14730 | Training loss: 0.3940\n",
            "Epoch 80/100 Iteration 14740 | Training loss: 0.4380\n",
            "Epoch 80/100 Iteration 14750 | Training loss: 0.4243\n",
            "Epoch 80/100 Iteration 14760 | Training loss: 0.4476\n",
            "Epoch 80/100 Iteration 14770 | Training loss: 0.3614\n",
            "Epoch 80/100 Iteration 14780 | Training loss: 0.3695\n",
            "Epoch 80/100 Iteration 14790 | Training loss: 0.3784\n",
            "Epoch 80/100 Iteration 14800 | Training loss: 0.4157\n",
            "Epoch 80/100 Iteration 14810 | Training loss: 0.4642\n",
            "Epoch 80/100 Iteration 14820 | Training loss: 0.3894\n",
            "Epoch 80/100 Iteration 14830 | Training loss: 0.4442\n",
            "Epoch 80/100 Iteration 14840 | Training loss: 0.4112\n",
            "Epoch 80/100 Iteration 14850 | Training loss: 0.3648\n",
            "Epoch 80/100 Iteration 14860 | Training loss: 0.4314\n",
            "Epoch 80/100 Iteration 14870 | Training loss: 0.3459\n",
            "Epoch 80/100 Iteration 14880 | Training loss: 0.4156\n",
            "Epoch 81/100 Iteration 14890 | Training loss: 0.4465\n",
            "Epoch 81/100 Iteration 14900 | Training loss: 0.4525\n",
            "Epoch 81/100 Iteration 14910 | Training loss: 0.3844\n",
            "Epoch 81/100 Iteration 14920 | Training loss: 0.4397\n",
            "Epoch 81/100 Iteration 14930 | Training loss: 0.4339\n",
            "Epoch 81/100 Iteration 14940 | Training loss: 0.3549\n",
            "Epoch 81/100 Iteration 14950 | Training loss: 0.4366\n",
            "Epoch 81/100 Iteration 14960 | Training loss: 0.4334\n",
            "Epoch 81/100 Iteration 14970 | Training loss: 0.3975\n",
            "Epoch 81/100 Iteration 14980 | Training loss: 0.3771\n",
            "Epoch 81/100 Iteration 14990 | Training loss: 0.3520\n",
            "Epoch 81/100 Iteration 15000 | Training loss: 0.4561\n",
            "Epoch 81/100 Iteration 15010 | Training loss: 0.3154\n",
            "Epoch 81/100 Iteration 15020 | Training loss: 0.3925\n",
            "Epoch 81/100 Iteration 15030 | Training loss: 0.4300\n",
            "Epoch 81/100 Iteration 15040 | Training loss: 0.4398\n",
            "Epoch 81/100 Iteration 15050 | Training loss: 0.5205\n",
            "Epoch 81/100 Iteration 15060 | Training loss: 0.3543\n",
            "Epoch 82/100 Iteration 15070 | Training loss: 0.4229\n",
            "Epoch 82/100 Iteration 15080 | Training loss: 0.3967\n",
            "Epoch 82/100 Iteration 15090 | Training loss: 0.4828\n",
            "Epoch 82/100 Iteration 15100 | Training loss: 0.4348\n",
            "Epoch 82/100 Iteration 15110 | Training loss: 0.5423\n",
            "Epoch 82/100 Iteration 15120 | Training loss: 0.4106\n",
            "Epoch 82/100 Iteration 15130 | Training loss: 0.3860\n",
            "Epoch 82/100 Iteration 15140 | Training loss: 0.3996\n",
            "Epoch 82/100 Iteration 15150 | Training loss: 0.4120\n",
            "Epoch 82/100 Iteration 15160 | Training loss: 0.4498\n",
            "Epoch 82/100 Iteration 15170 | Training loss: 0.3301\n",
            "Epoch 82/100 Iteration 15180 | Training loss: 0.3969\n",
            "Epoch 82/100 Iteration 15190 | Training loss: 0.4288\n",
            "Epoch 82/100 Iteration 15200 | Training loss: 0.3891\n",
            "Epoch 82/100 Iteration 15210 | Training loss: 0.4592\n",
            "Epoch 82/100 Iteration 15220 | Training loss: 0.3880\n",
            "Epoch 82/100 Iteration 15230 | Training loss: 0.4352\n",
            "Epoch 82/100 Iteration 15240 | Training loss: 0.4620\n",
            "Epoch 82/100 Iteration 15250 | Training loss: 0.3948\n",
            "Epoch 83/100 Iteration 15260 | Training loss: 0.4665\n",
            "Epoch 83/100 Iteration 15270 | Training loss: 0.2954\n",
            "Epoch 83/100 Iteration 15280 | Training loss: 0.4199\n",
            "Epoch 83/100 Iteration 15290 | Training loss: 0.4383\n",
            "Epoch 83/100 Iteration 15300 | Training loss: 0.4127\n",
            "Epoch 83/100 Iteration 15310 | Training loss: 0.4073\n",
            "Epoch 83/100 Iteration 15320 | Training loss: 0.4729\n",
            "Epoch 83/100 Iteration 15330 | Training loss: 0.3957\n",
            "Epoch 83/100 Iteration 15340 | Training loss: 0.3328\n",
            "Epoch 83/100 Iteration 15350 | Training loss: 0.3563\n",
            "Epoch 83/100 Iteration 15360 | Training loss: 0.4334\n",
            "Epoch 83/100 Iteration 15370 | Training loss: 0.4104\n",
            "Epoch 83/100 Iteration 15380 | Training loss: 0.4262\n",
            "Epoch 83/100 Iteration 15390 | Training loss: 0.3828\n",
            "Epoch 83/100 Iteration 15400 | Training loss: 0.4421\n",
            "Epoch 83/100 Iteration 15410 | Training loss: 0.3979\n",
            "Epoch 83/100 Iteration 15420 | Training loss: 0.5184\n",
            "Epoch 83/100 Iteration 15430 | Training loss: 0.3648\n",
            "Epoch 84/100 Iteration 15440 | Training loss: 0.3754\n",
            "Epoch 84/100 Iteration 15450 | Training loss: 0.4477\n",
            "Epoch 84/100 Iteration 15460 | Training loss: 0.4594\n",
            "Epoch 84/100 Iteration 15470 | Training loss: 0.4435\n",
            "Epoch 84/100 Iteration 15480 | Training loss: 0.4887\n",
            "Epoch 84/100 Iteration 15490 | Training loss: 0.3803\n",
            "Epoch 84/100 Iteration 15500 | Training loss: 0.4410\n",
            "Epoch 84/100 Iteration 15510 | Training loss: 0.3778\n",
            "Epoch 84/100 Iteration 15520 | Training loss: 0.3790\n",
            "Epoch 84/100 Iteration 15530 | Training loss: 0.4243\n",
            "Epoch 84/100 Iteration 15540 | Training loss: 0.3540\n",
            "Epoch 84/100 Iteration 15550 | Training loss: 0.3676\n",
            "Epoch 84/100 Iteration 15560 | Training loss: 0.4157\n",
            "Epoch 84/100 Iteration 15570 | Training loss: 0.3881\n",
            "Epoch 84/100 Iteration 15580 | Training loss: 0.4119\n",
            "Epoch 84/100 Iteration 15590 | Training loss: 0.3523\n",
            "Epoch 84/100 Iteration 15600 | Training loss: 0.3944\n",
            "Epoch 84/100 Iteration 15610 | Training loss: 0.4247\n",
            "Epoch 84/100 Iteration 15620 | Training loss: 0.3929\n",
            "Epoch 85/100 Iteration 15630 | Training loss: 0.4010\n",
            "Epoch 85/100 Iteration 15640 | Training loss: 0.4031\n",
            "Epoch 85/100 Iteration 15650 | Training loss: 0.4526\n",
            "Epoch 85/100 Iteration 15660 | Training loss: 0.3954\n",
            "Epoch 85/100 Iteration 15670 | Training loss: 0.4391\n",
            "Epoch 85/100 Iteration 15680 | Training loss: 0.4278\n",
            "Epoch 85/100 Iteration 15690 | Training loss: 0.4465\n",
            "Epoch 85/100 Iteration 15700 | Training loss: 0.3603\n",
            "Epoch 85/100 Iteration 15710 | Training loss: 0.3703\n",
            "Epoch 85/100 Iteration 15720 | Training loss: 0.3737\n",
            "Epoch 85/100 Iteration 15730 | Training loss: 0.4092\n",
            "Epoch 85/100 Iteration 15740 | Training loss: 0.4583\n",
            "Epoch 85/100 Iteration 15750 | Training loss: 0.3859\n",
            "Epoch 85/100 Iteration 15760 | Training loss: 0.4454\n",
            "Epoch 85/100 Iteration 15770 | Training loss: 0.4097\n",
            "Epoch 85/100 Iteration 15780 | Training loss: 0.3624\n",
            "Epoch 85/100 Iteration 15790 | Training loss: 0.4315\n",
            "Epoch 85/100 Iteration 15800 | Training loss: 0.3451\n",
            "Epoch 85/100 Iteration 15810 | Training loss: 0.4101\n",
            "Epoch 86/100 Iteration 15820 | Training loss: 0.4462\n",
            "Epoch 86/100 Iteration 15830 | Training loss: 0.4488\n",
            "Epoch 86/100 Iteration 15840 | Training loss: 0.3812\n",
            "Epoch 86/100 Iteration 15850 | Training loss: 0.4410\n",
            "Epoch 86/100 Iteration 15860 | Training loss: 0.4335\n",
            "Epoch 86/100 Iteration 15870 | Training loss: 0.3590\n",
            "Epoch 86/100 Iteration 15880 | Training loss: 0.4267\n",
            "Epoch 86/100 Iteration 15890 | Training loss: 0.4258\n",
            "Epoch 86/100 Iteration 15900 | Training loss: 0.3913\n",
            "Epoch 86/100 Iteration 15910 | Training loss: 0.3794\n",
            "Epoch 86/100 Iteration 15920 | Training loss: 0.3411\n",
            "Epoch 86/100 Iteration 15930 | Training loss: 0.4459\n",
            "Epoch 86/100 Iteration 15940 | Training loss: 0.3129\n",
            "Epoch 86/100 Iteration 15950 | Training loss: 0.3850\n",
            "Epoch 86/100 Iteration 15960 | Training loss: 0.4168\n",
            "Epoch 86/100 Iteration 15970 | Training loss: 0.4352\n",
            "Epoch 86/100 Iteration 15980 | Training loss: 0.5175\n",
            "Epoch 86/100 Iteration 15990 | Training loss: 0.3529\n",
            "Epoch 87/100 Iteration 16000 | Training loss: 0.4270\n",
            "Epoch 87/100 Iteration 16010 | Training loss: 0.3874\n",
            "Epoch 87/100 Iteration 16020 | Training loss: 0.4896\n",
            "Epoch 87/100 Iteration 16030 | Training loss: 0.4421\n",
            "Epoch 87/100 Iteration 16040 | Training loss: 0.5340\n",
            "Epoch 87/100 Iteration 16050 | Training loss: 0.4040\n",
            "Epoch 87/100 Iteration 16060 | Training loss: 0.3855\n",
            "Epoch 87/100 Iteration 16070 | Training loss: 0.4071\n",
            "Epoch 87/100 Iteration 16080 | Training loss: 0.4164\n",
            "Epoch 87/100 Iteration 16090 | Training loss: 0.4498\n",
            "Epoch 87/100 Iteration 16100 | Training loss: 0.3338\n",
            "Epoch 87/100 Iteration 16110 | Training loss: 0.3885\n",
            "Epoch 87/100 Iteration 16120 | Training loss: 0.4327\n",
            "Epoch 87/100 Iteration 16130 | Training loss: 0.3846\n",
            "Epoch 87/100 Iteration 16140 | Training loss: 0.4490\n",
            "Epoch 87/100 Iteration 16150 | Training loss: 0.3814\n",
            "Epoch 87/100 Iteration 16160 | Training loss: 0.4300\n",
            "Epoch 87/100 Iteration 16170 | Training loss: 0.4519\n",
            "Epoch 87/100 Iteration 16180 | Training loss: 0.3903\n",
            "Epoch 88/100 Iteration 16190 | Training loss: 0.4671\n",
            "Epoch 88/100 Iteration 16200 | Training loss: 0.2896\n",
            "Epoch 88/100 Iteration 16210 | Training loss: 0.4174\n",
            "Epoch 88/100 Iteration 16220 | Training loss: 0.4248\n",
            "Epoch 88/100 Iteration 16230 | Training loss: 0.4053\n",
            "Epoch 88/100 Iteration 16240 | Training loss: 0.4051\n",
            "Epoch 88/100 Iteration 16250 | Training loss: 0.4632\n",
            "Epoch 88/100 Iteration 16260 | Training loss: 0.3967\n",
            "Epoch 88/100 Iteration 16270 | Training loss: 0.3286\n",
            "Epoch 88/100 Iteration 16280 | Training loss: 0.3500\n",
            "Epoch 88/100 Iteration 16290 | Training loss: 0.4222\n",
            "Epoch 88/100 Iteration 16300 | Training loss: 0.4091\n",
            "Epoch 88/100 Iteration 16310 | Training loss: 0.4377\n",
            "Epoch 88/100 Iteration 16320 | Training loss: 0.3864\n",
            "Epoch 88/100 Iteration 16330 | Training loss: 0.4387\n",
            "Epoch 88/100 Iteration 16340 | Training loss: 0.3947\n",
            "Epoch 88/100 Iteration 16350 | Training loss: 0.5238\n",
            "Epoch 88/100 Iteration 16360 | Training loss: 0.3614\n",
            "Epoch 89/100 Iteration 16370 | Training loss: 0.3744\n",
            "Epoch 89/100 Iteration 16380 | Training loss: 0.4449\n",
            "Epoch 89/100 Iteration 16390 | Training loss: 0.4565\n",
            "Epoch 89/100 Iteration 16400 | Training loss: 0.4267\n",
            "Epoch 89/100 Iteration 16410 | Training loss: 0.4848\n",
            "Epoch 89/100 Iteration 16420 | Training loss: 0.3798\n",
            "Epoch 89/100 Iteration 16430 | Training loss: 0.4362\n",
            "Epoch 89/100 Iteration 16440 | Training loss: 0.3643\n",
            "Epoch 89/100 Iteration 16450 | Training loss: 0.3679\n",
            "Epoch 89/100 Iteration 16460 | Training loss: 0.4164\n",
            "Epoch 89/100 Iteration 16470 | Training loss: 0.3494\n",
            "Epoch 89/100 Iteration 16480 | Training loss: 0.3563\n",
            "Epoch 89/100 Iteration 16490 | Training loss: 0.4122\n",
            "Epoch 89/100 Iteration 16500 | Training loss: 0.3926\n",
            "Epoch 89/100 Iteration 16510 | Training loss: 0.4068\n",
            "Epoch 89/100 Iteration 16520 | Training loss: 0.3440\n",
            "Epoch 89/100 Iteration 16530 | Training loss: 0.3920\n",
            "Epoch 89/100 Iteration 16540 | Training loss: 0.4221\n",
            "Epoch 89/100 Iteration 16550 | Training loss: 0.3967\n",
            "Epoch 90/100 Iteration 16560 | Training loss: 0.3917\n",
            "Epoch 90/100 Iteration 16570 | Training loss: 0.3959\n",
            "Epoch 90/100 Iteration 16580 | Training loss: 0.4496\n",
            "Epoch 90/100 Iteration 16590 | Training loss: 0.3944\n",
            "Epoch 90/100 Iteration 16600 | Training loss: 0.4266\n",
            "Epoch 90/100 Iteration 16610 | Training loss: 0.4200\n",
            "Epoch 90/100 Iteration 16620 | Training loss: 0.4347\n",
            "Epoch 90/100 Iteration 16630 | Training loss: 0.3541\n",
            "Epoch 90/100 Iteration 16640 | Training loss: 0.3686\n",
            "Epoch 90/100 Iteration 16650 | Training loss: 0.3670\n",
            "Epoch 90/100 Iteration 16660 | Training loss: 0.4040\n",
            "Epoch 90/100 Iteration 16670 | Training loss: 0.4529\n",
            "Epoch 90/100 Iteration 16680 | Training loss: 0.3854\n",
            "Epoch 90/100 Iteration 16690 | Training loss: 0.4343\n",
            "Epoch 90/100 Iteration 16700 | Training loss: 0.4073\n",
            "Epoch 90/100 Iteration 16710 | Training loss: 0.3592\n",
            "Epoch 90/100 Iteration 16720 | Training loss: 0.4290\n",
            "Epoch 90/100 Iteration 16730 | Training loss: 0.3401\n",
            "Epoch 90/100 Iteration 16740 | Training loss: 0.4036\n",
            "Epoch 91/100 Iteration 16750 | Training loss: 0.4464\n",
            "Epoch 91/100 Iteration 16760 | Training loss: 0.4483\n",
            "Epoch 91/100 Iteration 16770 | Training loss: 0.3776\n",
            "Epoch 91/100 Iteration 16780 | Training loss: 0.4394\n",
            "Epoch 91/100 Iteration 16790 | Training loss: 0.4284\n",
            "Epoch 91/100 Iteration 16800 | Training loss: 0.3493\n",
            "Epoch 91/100 Iteration 16810 | Training loss: 0.4242\n",
            "Epoch 91/100 Iteration 16820 | Training loss: 0.4212\n",
            "Epoch 91/100 Iteration 16830 | Training loss: 0.3906\n",
            "Epoch 91/100 Iteration 16840 | Training loss: 0.3720\n",
            "Epoch 91/100 Iteration 16850 | Training loss: 0.3412\n",
            "Epoch 91/100 Iteration 16860 | Training loss: 0.4466\n",
            "Epoch 91/100 Iteration 16870 | Training loss: 0.3109\n",
            "Epoch 91/100 Iteration 16880 | Training loss: 0.3821\n",
            "Epoch 91/100 Iteration 16890 | Training loss: 0.4211\n",
            "Epoch 91/100 Iteration 16900 | Training loss: 0.4305\n",
            "Epoch 91/100 Iteration 16910 | Training loss: 0.5109\n",
            "Epoch 91/100 Iteration 16920 | Training loss: 0.3449\n",
            "Epoch 92/100 Iteration 16930 | Training loss: 0.4160\n",
            "Epoch 92/100 Iteration 16940 | Training loss: 0.3871\n",
            "Epoch 92/100 Iteration 16950 | Training loss: 0.4766\n",
            "Epoch 92/100 Iteration 16960 | Training loss: 0.4290\n",
            "Epoch 92/100 Iteration 16970 | Training loss: 0.5327\n",
            "Epoch 92/100 Iteration 16980 | Training loss: 0.4034\n",
            "Epoch 92/100 Iteration 16990 | Training loss: 0.3809\n",
            "Epoch 92/100 Iteration 17000 | Training loss: 0.3972\n",
            "Epoch 92/100 Iteration 17010 | Training loss: 0.4096\n",
            "Epoch 92/100 Iteration 17020 | Training loss: 0.4403\n",
            "Epoch 92/100 Iteration 17030 | Training loss: 0.3284\n",
            "Epoch 92/100 Iteration 17040 | Training loss: 0.3897\n",
            "Epoch 92/100 Iteration 17050 | Training loss: 0.4229\n",
            "Epoch 92/100 Iteration 17060 | Training loss: 0.3732\n",
            "Epoch 92/100 Iteration 17070 | Training loss: 0.4423\n",
            "Epoch 92/100 Iteration 17080 | Training loss: 0.3770\n",
            "Epoch 92/100 Iteration 17090 | Training loss: 0.4265\n",
            "Epoch 92/100 Iteration 17100 | Training loss: 0.4516\n",
            "Epoch 92/100 Iteration 17110 | Training loss: 0.3848\n",
            "Epoch 93/100 Iteration 17120 | Training loss: 0.4711\n",
            "Epoch 93/100 Iteration 17130 | Training loss: 0.2893\n",
            "Epoch 93/100 Iteration 17140 | Training loss: 0.4126\n",
            "Epoch 93/100 Iteration 17150 | Training loss: 0.4292\n",
            "Epoch 93/100 Iteration 17160 | Training loss: 0.3989\n",
            "Epoch 93/100 Iteration 17170 | Training loss: 0.3990\n",
            "Epoch 93/100 Iteration 17180 | Training loss: 0.4617\n",
            "Epoch 93/100 Iteration 17190 | Training loss: 0.3864\n",
            "Epoch 93/100 Iteration 17200 | Training loss: 0.3274\n",
            "Epoch 93/100 Iteration 17210 | Training loss: 0.3515\n",
            "Epoch 93/100 Iteration 17220 | Training loss: 0.4165\n",
            "Epoch 93/100 Iteration 17230 | Training loss: 0.4062\n",
            "Epoch 93/100 Iteration 17240 | Training loss: 0.4118\n",
            "Epoch 93/100 Iteration 17250 | Training loss: 0.3786\n",
            "Epoch 93/100 Iteration 17260 | Training loss: 0.4286\n",
            "Epoch 93/100 Iteration 17270 | Training loss: 0.3846\n",
            "Epoch 93/100 Iteration 17280 | Training loss: 0.5126\n",
            "Epoch 93/100 Iteration 17290 | Training loss: 0.3624\n",
            "Epoch 94/100 Iteration 17300 | Training loss: 0.3782\n",
            "Epoch 94/100 Iteration 17310 | Training loss: 0.4449\n",
            "Epoch 94/100 Iteration 17320 | Training loss: 0.4540\n",
            "Epoch 94/100 Iteration 17330 | Training loss: 0.4280\n",
            "Epoch 94/100 Iteration 17340 | Training loss: 0.4757\n",
            "Epoch 94/100 Iteration 17350 | Training loss: 0.3703\n",
            "Epoch 94/100 Iteration 17360 | Training loss: 0.4291\n",
            "Epoch 94/100 Iteration 17370 | Training loss: 0.3635\n",
            "Epoch 94/100 Iteration 17380 | Training loss: 0.3632\n",
            "Epoch 94/100 Iteration 17390 | Training loss: 0.4155\n",
            "Epoch 94/100 Iteration 17400 | Training loss: 0.3490\n",
            "Epoch 94/100 Iteration 17410 | Training loss: 0.3569\n",
            "Epoch 94/100 Iteration 17420 | Training loss: 0.4151\n",
            "Epoch 94/100 Iteration 17430 | Training loss: 0.3863\n",
            "Epoch 94/100 Iteration 17440 | Training loss: 0.4135\n",
            "Epoch 94/100 Iteration 17450 | Training loss: 0.3426\n",
            "Epoch 94/100 Iteration 17460 | Training loss: 0.3897\n",
            "Epoch 94/100 Iteration 17470 | Training loss: 0.4165\n",
            "Epoch 94/100 Iteration 17480 | Training loss: 0.3906\n",
            "Epoch 95/100 Iteration 17490 | Training loss: 0.3894\n",
            "Epoch 95/100 Iteration 17500 | Training loss: 0.3914\n",
            "Epoch 95/100 Iteration 17510 | Training loss: 0.4448\n",
            "Epoch 95/100 Iteration 17520 | Training loss: 0.3864\n",
            "Epoch 95/100 Iteration 17530 | Training loss: 0.4251\n",
            "Epoch 95/100 Iteration 17540 | Training loss: 0.4088\n",
            "Epoch 95/100 Iteration 17550 | Training loss: 0.4366\n",
            "Epoch 95/100 Iteration 17560 | Training loss: 0.3549\n",
            "Epoch 95/100 Iteration 17570 | Training loss: 0.3624\n",
            "Epoch 95/100 Iteration 17580 | Training loss: 0.3724\n",
            "Epoch 95/100 Iteration 17590 | Training loss: 0.4070\n",
            "Epoch 95/100 Iteration 17600 | Training loss: 0.4434\n",
            "Epoch 95/100 Iteration 17610 | Training loss: 0.3823\n",
            "Epoch 95/100 Iteration 17620 | Training loss: 0.4305\n",
            "Epoch 95/100 Iteration 17630 | Training loss: 0.4042\n",
            "Epoch 95/100 Iteration 17640 | Training loss: 0.3533\n",
            "Epoch 95/100 Iteration 17650 | Training loss: 0.4210\n",
            "Epoch 95/100 Iteration 17660 | Training loss: 0.3367\n",
            "Epoch 95/100 Iteration 17670 | Training loss: 0.4034\n",
            "Epoch 96/100 Iteration 17680 | Training loss: 0.4481\n",
            "Epoch 96/100 Iteration 17690 | Training loss: 0.4450\n",
            "Epoch 96/100 Iteration 17700 | Training loss: 0.3768\n",
            "Epoch 96/100 Iteration 17710 | Training loss: 0.4291\n",
            "Epoch 96/100 Iteration 17720 | Training loss: 0.4275\n",
            "Epoch 96/100 Iteration 17730 | Training loss: 0.3472\n",
            "Epoch 96/100 Iteration 17740 | Training loss: 0.4144\n",
            "Epoch 96/100 Iteration 17750 | Training loss: 0.4176\n",
            "Epoch 96/100 Iteration 17760 | Training loss: 0.3818\n",
            "Epoch 96/100 Iteration 17770 | Training loss: 0.3707\n",
            "Epoch 96/100 Iteration 17780 | Training loss: 0.3423\n",
            "Epoch 96/100 Iteration 17790 | Training loss: 0.4405\n",
            "Epoch 96/100 Iteration 17800 | Training loss: 0.3060\n",
            "Epoch 96/100 Iteration 17810 | Training loss: 0.3814\n",
            "Epoch 96/100 Iteration 17820 | Training loss: 0.4183\n",
            "Epoch 96/100 Iteration 17830 | Training loss: 0.4287\n",
            "Epoch 96/100 Iteration 17840 | Training loss: 0.5130\n",
            "Epoch 96/100 Iteration 17850 | Training loss: 0.3441\n",
            "Epoch 97/100 Iteration 17860 | Training loss: 0.4141\n",
            "Epoch 97/100 Iteration 17870 | Training loss: 0.3852\n",
            "Epoch 97/100 Iteration 17880 | Training loss: 0.4697\n",
            "Epoch 97/100 Iteration 17890 | Training loss: 0.4237\n",
            "Epoch 97/100 Iteration 17900 | Training loss: 0.5200\n",
            "Epoch 97/100 Iteration 17910 | Training loss: 0.4034\n",
            "Epoch 97/100 Iteration 17920 | Training loss: 0.3747\n",
            "Epoch 97/100 Iteration 17930 | Training loss: 0.3959\n",
            "Epoch 97/100 Iteration 17940 | Training loss: 0.4090\n",
            "Epoch 97/100 Iteration 17950 | Training loss: 0.4300\n",
            "Epoch 97/100 Iteration 17960 | Training loss: 0.3249\n",
            "Epoch 97/100 Iteration 17970 | Training loss: 0.3839\n",
            "Epoch 97/100 Iteration 17980 | Training loss: 0.4138\n",
            "Epoch 97/100 Iteration 17990 | Training loss: 0.3709\n",
            "Epoch 97/100 Iteration 18000 | Training loss: 0.4426\n",
            "Epoch 97/100 Iteration 18010 | Training loss: 0.3733\n",
            "Epoch 97/100 Iteration 18020 | Training loss: 0.4207\n",
            "Epoch 97/100 Iteration 18030 | Training loss: 0.4504\n",
            "Epoch 97/100 Iteration 18040 | Training loss: 0.3805\n",
            "Epoch 98/100 Iteration 18050 | Training loss: 0.4650\n",
            "Epoch 98/100 Iteration 18060 | Training loss: 0.2848\n",
            "Epoch 98/100 Iteration 18070 | Training loss: 0.4103\n",
            "Epoch 98/100 Iteration 18080 | Training loss: 0.4235\n",
            "Epoch 98/100 Iteration 18090 | Training loss: 0.3979\n",
            "Epoch 98/100 Iteration 18100 | Training loss: 0.3991\n",
            "Epoch 98/100 Iteration 18110 | Training loss: 0.4639\n",
            "Epoch 98/100 Iteration 18120 | Training loss: 0.3855\n",
            "Epoch 98/100 Iteration 18130 | Training loss: 0.3247\n",
            "Epoch 98/100 Iteration 18140 | Training loss: 0.3428\n",
            "Epoch 98/100 Iteration 18150 | Training loss: 0.4171\n",
            "Epoch 98/100 Iteration 18160 | Training loss: 0.3991\n",
            "Epoch 98/100 Iteration 18170 | Training loss: 0.4105\n",
            "Epoch 98/100 Iteration 18180 | Training loss: 0.3717\n",
            "Epoch 98/100 Iteration 18190 | Training loss: 0.4247\n",
            "Epoch 98/100 Iteration 18200 | Training loss: 0.3868\n",
            "Epoch 98/100 Iteration 18210 | Training loss: 0.5138\n",
            "Epoch 98/100 Iteration 18220 | Training loss: 0.3529\n",
            "Epoch 99/100 Iteration 18230 | Training loss: 0.3693\n",
            "Epoch 99/100 Iteration 18240 | Training loss: 0.4467\n",
            "Epoch 99/100 Iteration 18250 | Training loss: 0.4610\n",
            "Epoch 99/100 Iteration 18260 | Training loss: 0.4293\n",
            "Epoch 99/100 Iteration 18270 | Training loss: 0.4747\n",
            "Epoch 99/100 Iteration 18280 | Training loss: 0.3675\n",
            "Epoch 99/100 Iteration 18290 | Training loss: 0.4230\n",
            "Epoch 99/100 Iteration 18300 | Training loss: 0.3499\n",
            "Epoch 99/100 Iteration 18310 | Training loss: 0.3538\n",
            "Epoch 99/100 Iteration 18320 | Training loss: 0.4098\n",
            "Epoch 99/100 Iteration 18330 | Training loss: 0.3454\n",
            "Epoch 99/100 Iteration 18340 | Training loss: 0.3498\n",
            "Epoch 99/100 Iteration 18350 | Training loss: 0.4059\n",
            "Epoch 99/100 Iteration 18360 | Training loss: 0.3793\n",
            "Epoch 99/100 Iteration 18370 | Training loss: 0.4000\n",
            "Epoch 99/100 Iteration 18380 | Training loss: 0.3342\n",
            "Epoch 99/100 Iteration 18390 | Training loss: 0.3831\n",
            "Epoch 99/100 Iteration 18400 | Training loss: 0.4130\n",
            "Epoch 99/100 Iteration 18410 | Training loss: 0.3840\n",
            "Epoch 100/100 Iteration 18420 | Training loss: 0.3877\n",
            "Epoch 100/100 Iteration 18430 | Training loss: 0.3884\n",
            "Epoch 100/100 Iteration 18440 | Training loss: 0.4395\n",
            "Epoch 100/100 Iteration 18450 | Training loss: 0.3858\n",
            "Epoch 100/100 Iteration 18460 | Training loss: 0.4287\n",
            "Epoch 100/100 Iteration 18470 | Training loss: 0.4243\n",
            "Epoch 100/100 Iteration 18480 | Training loss: 0.4300\n",
            "Epoch 100/100 Iteration 18490 | Training loss: 0.3485\n",
            "Epoch 100/100 Iteration 18500 | Training loss: 0.3556\n",
            "Epoch 100/100 Iteration 18510 | Training loss: 0.3661\n",
            "Epoch 100/100 Iteration 18520 | Training loss: 0.4040\n",
            "Epoch 100/100 Iteration 18530 | Training loss: 0.4405\n",
            "Epoch 100/100 Iteration 18540 | Training loss: 0.3847\n",
            "Epoch 100/100 Iteration 18550 | Training loss: 0.4242\n",
            "Epoch 100/100 Iteration 18560 | Training loss: 0.4041\n",
            "Epoch 100/100 Iteration 18570 | Training loss: 0.3518\n",
            "Epoch 100/100 Iteration 18580 | Training loss: 0.4191\n",
            "Epoch 100/100 Iteration 18590 | Training loss: 0.3376\n",
            "Epoch 100/100 Iteration 18600 | Training loss: 0.3958\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "y04DqaiKrO9L",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 550
        },
        "outputId": "0aaa4d62-6296-4207-d70a-3d2da23e6fe8"
      },
      "cell_type": "code",
      "source": [
        "# Char RNN model in the sampling mode\n",
        "np.random.seed(123)\n",
        "\n",
        "rnn=CharRNN(len(chars), sampling=True)\n",
        "print(rnn.sample(ckpt_dir='./model-100/', output_length=1000))"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " << lstm_outputs >>  Tensor(\"rnn/transpose_1:0\", shape=(1, 1, 128), dtype=float32)\n",
            "INFO:tensorflow:Restoring parameters from ./model-100/language_modeling.ckpt\n",
            "The thent wither\">\n",
            "      <strec  therke=\"diverpentt/searse. bongright deathing suppite-sont ating-suget to ting\"epares=\"itanne\">\n",
            "\n",
            "  </detanath>\n",
            "  </div>\n",
            "\n",
            "    </div>\n",
            "\n",
            "    <li class=\"brom roun dosentex\">\n",
            "     </details-lise-jumc=\"je-cope-tocrens-rent drepp-sitere\">\n",
            "  \n",
            "      </span>\n",
            "      <a dit=\"jsp-jump-to-bence- timuss-cript\"></stan data-ineanor=\"repo_lorker\">\n",
            "      <span class=\"jums-contio-ting reponitoun or-farl time dealle,</td>\n",
            "      </tr>\n",
            "      <tr>\n",
            "        <td id=\"L381\" class=\"blob-num js-line-number\" data-line-number=\"568\"></td>\n",
            "        <td id=\"LC568\" class=\"blob-code blob-code-inner js-file-line\">\n",
            "</td>\n",
            "      </tr>\n",
            "      <tr>\n",
            "        <td id=\"L4012\" class=\"blob-num js-line-number\" data-line-number=\"2041\"></td>\n",
            "        <td id=\"LC2022\" class=\"blob-code blob-code-inner js-file-line\">   Ham. How to sis of a my bonters of the thought</td>\n",
            "      </tr>\n",
            "      <tr>\n",
            "        <td id=\"L2238\" class=\"blob-num js-line-number\" data-line-number=\"1328\"></td>\n",
            "        <td id=\"LC4248\" class=\"blob-code blo\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "6lNH8JfW_Ucd",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}