breakout_natureQDN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "#from generator import UNETGenerator\n",
    "from keras.layers import Activation, Input, Dropout, merge,concatenate\n",
    "from keras.layers.convolutional import Convolution2D, UpSampling2D\n",
    "from keras.layers import Conv2D,Conv2DTranspose,Flatten,Dense\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.advanced_activations import LeakyReLU,ReLU\n",
    "from keras.models import Model\n",
    "import cv2\n",
    "from collections import deque\n",
    "from keras.optimizers import Adam\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import random\n",
    "import keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.compat.v1 import ConfigProto\n",
    "from tensorflow.compat.v1 import InteractiveSession\n",
    "config = ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "session = InteractiveSession(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyper Parameters for DQN\n",
    "GAMMA = 0.9 # discount factor for target Q\n",
    "INITIAL_EPSILON = 0.5 # starting value of epsilon\n",
    "FINAL_EPSILON = 0.01 # final value of epsilon\n",
    "REPLAY_SIZE = 5000 # experience replay buffer size\n",
    "BATCH_SIZE = 32 # size of minibatch\n",
    "IMG_SIZE = 80"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(img):\n",
    "    img_temp = img.mean(axis = 2)\n",
    "#    img_temp = cv2.resize(img_temp,(IMG_SIZE,IMG_SIZE))\n",
    "    x = -1\n",
    "    y = -1\n",
    "    flag = 0\n",
    "    if len(np.where((img_temp[100:189,8:152])!= 0)[0]) != 0:\n",
    "        x = np.where((img_temp[100:189,8:152])!= 0)[0][0]\n",
    "        y = np.where((img_temp[100:189,8:152])!= 0)[1][0]\n",
    "    if len(np.where((img_temp[193:,8:152])!= 0)[0]) != 0:\n",
    "        x = np.where((img_temp[193:,8:152])!= 0)[0][0] + 93\n",
    "        y = np.where((img_temp[193:,8:152])!= 0)[1][0]\n",
    "        flag = 1\n",
    "#         x = -2\n",
    "#         y = -2\n",
    "    p = int(np.where(img_temp[191:193,8:152])[1].mean() - 7.5)\n",
    "    #return img_temp\n",
    "    return (x,y,p,flag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ---------------------------------------------------------\n",
    "# Hyper Parameters\n",
    "ENV_NAME = 'Breakout-v0'\n",
    "EPISODE = 15000 # Episode limitation\n",
    "STEP = 500 # Step limitation in an episode\n",
    "TEST = 10 # The number of experiment test every 100 episode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN():\n",
    "  # DQN Agent\n",
    "  def __init__(self, env):\n",
    "    # init experience replay\n",
    "    self.replay_buffer = deque()\n",
    "    # init some parameters\n",
    "    self.time_step = 0\n",
    "    self.epsilon = INITIAL_EPSILON\n",
    "    self.learning_rate = 0.0001\n",
    "    #self.state_dim = env.observation_space.shape[0]\n",
    "    self.state_dim = (IMG_SIZE,IMG_SIZE)\n",
    "    self.action_dim = env.action_space.n - 1 \n",
    "\n",
    "    self.create_Q_network()\n",
    "    #self.create_training_method()\n",
    "\n",
    "    # Init session\n",
    "#     self.session = tf.InteractiveSession()\n",
    "#     self.session.run(tf.initialize_all_variables())\n",
    "\n",
    "# 更改state为：\n",
    "#    - 只使用屏幕的下半部分。（先不去关心具体撞击分数，只要接住）\n",
    "#    - 连续两帧球的轨迹，只有球在下半部分再行动。 \n",
    "#    - (x1,y1,x2,y2,p) p是平台的位置\n",
    "  \n",
    "  def create_Q_network(self):\n",
    "    # network weights\n",
    "    input_layer = Input(shape=(5,), name=\"unet_input\")\n",
    "    fc1 = Dense(128)(input_layer)\n",
    "    fc1 = LeakyReLU(alpha=0.1)(fc1)\n",
    "    fc2 = Dense(64)(fc1)\n",
    "    fc2 = LeakyReLU(alpha=0.1)(fc2)\n",
    "    fc3 = Dense(16)(fc2)\n",
    "    fc3 = LeakyReLU(alpha=0.1)(fc3)\n",
    "    #get Q_value\n",
    "    q_value= Dense(self.action_dim)(fc3)\n",
    "    self.model = Model(input=[input_layer], output=[q_value], name='Q_net')\n",
    "    self.model.compile(loss='mse',optimizer=Adam(lr=self.learning_rate))\n",
    "\n",
    "    # target net\n",
    "    self.target_model = keras.models.clone_model(self.model)\n",
    "    self.target_model.set_weights(self.model.get_weights())\n",
    "\n",
    "  def update_target(self):\n",
    "    self.target_model.set_weights(self.model.get_weights())\n",
    "    \n",
    "  def perceive(self,state,action,reward,next_state,done):\n",
    "    next_q_value = self.target_model.predict(np.expand_dims(next_state,0))\n",
    "    self.replay_buffer.append((state,action,reward,next_q_value[0],done))\n",
    "    if len(self.replay_buffer) > REPLAY_SIZE:\n",
    "      self.replay_buffer.popleft()\n",
    "\n",
    "    if len(self.replay_buffer) > BATCH_SIZE:\n",
    "      return self.train_Q_network()\n",
    "\n",
    "  def train_Q_network(self):\n",
    "    self.time_step += 1\n",
    "    # Step 1: obtain random minibatch from replay memory\n",
    "    minibatch = random.sample(self.replay_buffer,BATCH_SIZE)\n",
    "    state_batch = np.array([data[0] for data in minibatch])\n",
    "    action_batch = np.array([data[1] for data in minibatch])\n",
    "    reward_batch = np.array([data[2] for data in minibatch])\n",
    "    next_q_batch = np.array([data[3] for data in minibatch])\n",
    "\n",
    "    # Step 2: calculate y\n",
    "    y_batch = []\n",
    "    for i in range(0,BATCH_SIZE):\n",
    "      done = minibatch[i][4]\n",
    "      if done:\n",
    "        y_batch.append(reward_batch[i])\n",
    "      else :\n",
    "        y_batch.append(reward_batch[i] + GAMMA * np.max(next_q_batch[i]))\n",
    "\n",
    "    # 获得训练的target\n",
    "    \n",
    "    label_f = self.model.predict(state_batch)\n",
    "    for i in range(0,BATCH_SIZE):\n",
    "        label_f[i][action_batch[i]-1] = y_batch[i]\n",
    "    \n",
    "    return self.model.fit(state_batch,label_f)\n",
    "    \n",
    "  def egreedy_action(self,state):\n",
    "    if state[0] < 0:\n",
    "        return 1;\n",
    "    Q_value = self.model.predict(np.expand_dims(state,0))[0]\n",
    "    if random.random() <= self.epsilon:\n",
    "      return random.randint(0,self.action_dim - 1) + 1\n",
    "    else:\n",
    "      return np.argmax(Q_value) + 1\n",
    "\n",
    "  def action(self,state):\n",
    "    if state[0] < 0:\n",
    "        return 1;\n",
    "    return np.argmax(self.model.predict(np.expand_dims(state,0))[0]) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython import display\n",
    "def main(env):\n",
    "  # initialize OpenAI Gym env and dqn agent\n",
    "\n",
    "#   agent.model.summary()\n",
    "#   return \n",
    "  for episode in range(EPISODE):\n",
    "    # initialize task\n",
    "    log_file = open(\"m_log2.log\",\"a\")\n",
    "    state = env.reset()\n",
    "    (x1,y1,p1,flag) = preprocess(state)\n",
    "    # Train\n",
    "    state_shadow = np.array([x1,y1,x1,y1,p1])\n",
    "    total_loss = 0 \n",
    "    total_reward = 0\n",
    "    for step in range(STEP):\n",
    "        print(\"episode:%d step:%d\" % (episode,step))\n",
    "        action = agent.egreedy_action(state_shadow)    \n",
    "        next_state,reward,done,_ = env.step(action)\n",
    "        (x2,y2,p2,flag) = preprocess(next_state)\n",
    "        next_state_shadow = np.array([x1,y1,x2,y2,p2])\n",
    "        # Define reward for agent\n",
    "        #reward_agent = -1 if done else 0.1\n",
    "        #加大落下的惩罚\n",
    "        if flag == 1:\n",
    "            reward = -10\n",
    "            done = True\n",
    "        else:\n",
    "            reward = 0.1\n",
    "        agent.perceive(state_shadow,action,reward,next_state_shadow,done)\n",
    "#         if cur_loss is not None:\n",
    "#             total_loss += cur_loss\n",
    "        total_reward += reward\n",
    "        state_shadow = next_state_shadow\n",
    "        x1,y1,p1 = x2,y2,p2\n",
    "        if done:\n",
    "            break\n",
    "    agent.epsilon -= (INITIAL_EPSILON - FINAL_EPSILON)/EPISODE\n",
    "    print(\"total_loss:{} total_reward:{}\".format(total_loss,total_reward))\n",
    "    # Test every 1000 episodes\n",
    "    \n",
    "    if episode % 10 == 0:\n",
    "        agent.update_target()\n",
    "        \n",
    "    if episode % 100 == 0:\n",
    "        display.clear_output(wait=True)\n",
    "        agent.model.save_weights(\"models/model2_{}.h5\".format(episode))\n",
    "        total_reward = 0\n",
    "        for i in range(TEST):\n",
    "            state = env.reset()\n",
    "            (x1,y1,p1,flag) = preprocess(state)\n",
    "            state_shadow = np.array([x1,y1,x1,y1,p1])\n",
    "            for j in range(STEP):\n",
    "                #env.render(mode='rgb_array')\n",
    "                action = agent.action(state_shadow) # direct action for test\n",
    "                next_state,reward,done,_ = env.step(action)\n",
    "                #state = preprocess(state)\n",
    "                (x2,y2,p2,flag) = preprocess(next_state)\n",
    "                if flag == 1:\n",
    "                    reward = -10\n",
    "                    done = True\n",
    "                else:\n",
    "                    reward = 0.1\n",
    "                next_state_shadow = np.array([x1,y1,x2,y2,p2])\n",
    "                total_reward += reward\n",
    "                state_shadow = next_state_shadow\n",
    "                x1,y1,p1 = x2,y2,p2\n",
    "                if done:\n",
    "                    break\n",
    "        ave_reward = total_reward/TEST\n",
    "        log_file.write('episode: {} Evaluation Average Reward: {}\\r\\n'.format(episode,ave_reward))\n",
    "        if ave_reward >= 280:\n",
    "            break\n",
    "    log_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:37: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"Q_net\", inputs=[<tf.Tenso..., outputs=[<tf.Tenso...)`\n"
     ]
    }
   ],
   "source": [
    "env = gym.make(ENV_NAME)\n",
    "agent = DQN(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:1901 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 83us/step - loss: 0.0461\n",
      "episode:1901 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0684\n",
      "episode:1901 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0275\n",
      "episode:1901 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0558\n",
      "episode:1901 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0689\n",
      "episode:1901 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.3975\n",
      "episode:1901 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.1113\n",
      "episode:1901 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0418\n",
      "episode:1901 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0142\n",
      "episode:1901 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.1516\n",
      "episode:1901 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0267\n",
      "episode:1901 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.2016\n",
      "episode:1901 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0921\n",
      "episode:1901 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0868\n",
      "episode:1901 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0272\n",
      "episode:1901 step:15\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0440\n",
      "episode:1901 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0401\n",
      "episode:1901 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0987\n",
      "episode:1901 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 80us/step - loss: 0.1498\n",
      "episode:1901 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.0939\n",
      "episode:1901 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 82us/step - loss: 0.0426\n",
      "episode:1901 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.0158\n",
      "episode:1901 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.0748\n",
      "episode:1901 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 86us/step - loss: 0.0495\n",
      "episode:1901 step:24\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 85us/step - loss: 0.0667\n",
      "total_loss:0 total_reward:-7.6\n",
      "episode:1902 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.1717\n",
      "episode:1902 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 84us/step - loss: 0.0176\n",
      "episode:1902 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.0316\n",
      "episode:1902 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 79us/step - loss: 0.0297\n",
      "episode:1902 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 86us/step - loss: 0.0674\n",
      "episode:1902 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 84us/step - loss: 0.0244\n",
      "episode:1902 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 83us/step - loss: 0.4253\n",
      "episode:1902 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 87us/step - loss: 0.0500\n",
      "episode:1902 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.7807\n",
      "episode:1902 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 83us/step - loss: 0.1233\n",
      "episode:1902 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 84us/step - loss: 0.0747\n",
      "episode:1902 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 87us/step - loss: 0.0212\n",
      "episode:1902 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 84us/step - loss: 0.0768\n",
      "episode:1902 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.0115\n",
      "episode:1902 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 86us/step - loss: 0.0336\n",
      "episode:1902 step:15\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.0864\n",
      "episode:1902 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 83us/step - loss: 0.0153\n",
      "episode:1902 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 86us/step - loss: 0.0513\n",
      "episode:1902 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 89us/step - loss: 0.0961\n",
      "episode:1902 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 92us/step - loss: 0.0392\n",
      "episode:1902 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.1089\n",
      "episode:1902 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.3475\n",
      "episode:1902 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 82us/step - loss: 0.5676\n",
      "episode:1902 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 84us/step - loss: 0.1610\n",
      "episode:1902 step:24\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.1641\n",
      "episode:1902 step:25\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 86us/step - loss: 0.2637\n",
      "episode:1902 step:26\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 83us/step - loss: 0.0487\n",
      "episode:1902 step:27\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.0878\n",
      "episode:1902 step:28\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 89us/step - loss: 0.2601\n",
      "episode:1902 step:29\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 84us/step - loss: 0.0845\n",
      "episode:1902 step:30\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.0519\n",
      "episode:1902 step:31\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 82us/step - loss: 0.2269\n",
      "episode:1902 step:32\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 84us/step - loss: 0.0314\n",
      "episode:1902 step:33\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.0608\n",
      "episode:1902 step:34\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.0248\n",
      "episode:1902 step:35\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 87us/step - loss: 0.0441\n",
      "episode:1902 step:36\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 94us/step - loss: 0.1083\n",
      "episode:1902 step:37\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 84us/step - loss: 0.0262\n",
      "episode:1902 step:38\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 79us/step - loss: 0.4097\n",
      "episode:1902 step:39\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 82us/step - loss: 0.2088\n",
      "episode:1902 step:40\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 86us/step - loss: 0.3029\n",
      "episode:1902 step:41\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 84us/step - loss: 0.0166\n",
      "episode:1902 step:42\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 84us/step - loss: 0.1824\n",
      "episode:1902 step:43\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.0304\n",
      "episode:1902 step:44\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 83us/step - loss: 0.0755\n",
      "episode:1902 step:45\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 84us/step - loss: 0.0173\n",
      "episode:1902 step:46\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 88us/step - loss: 0.2027\n",
      "episode:1902 step:47\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.1955\n",
      "episode:1902 step:48\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.0379\n",
      "episode:1902 step:49\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 83us/step - loss: 0.2186\n",
      "episode:1902 step:50\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 84us/step - loss: 0.1295\n",
      "episode:1902 step:51\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.1633\n",
      "episode:1902 step:52\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.0424\n",
      "episode:1902 step:53\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 89us/step - loss: 0.1291\n",
      "episode:1902 step:54\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 85us/step - loss: 0.0851\n",
      "episode:1902 step:55\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 84us/step - loss: 0.8982\n",
      "episode:1902 step:56\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 90us/step - loss: 0.3387\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:1902 step:57\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 87us/step - loss: 0.1482\n",
      "episode:1902 step:58\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 85us/step - loss: 0.0484\n",
      "episode:1902 step:59\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 93us/step - loss: 0.0320\n",
      "episode:1902 step:60\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 88us/step - loss: 0.1700\n",
      "episode:1902 step:61\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 84us/step - loss: 0.0763\n",
      "episode:1902 step:62\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 83us/step - loss: 0.0261\n",
      "episode:1902 step:63\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 80us/step - loss: 0.0173\n",
      "episode:1902 step:64\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 83us/step - loss: 0.0235\n",
      "episode:1902 step:65\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.0159\n",
      "total_loss:0 total_reward:-3.500000000000007\n",
      "episode:1903 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 79us/step - loss: 0.1741\n",
      "episode:1903 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 83us/step - loss: 0.0268\n",
      "episode:1903 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 84us/step - loss: 0.0174\n",
      "episode:1903 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 83us/step - loss: 0.1575\n",
      "episode:1903 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 88us/step - loss: 0.0594\n",
      "episode:1903 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.2841\n",
      "episode:1903 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 83us/step - loss: 0.0583\n",
      "episode:1903 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 88us/step - loss: 0.0508\n",
      "episode:1903 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.0605\n",
      "episode:1903 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 82us/step - loss: 0.0337\n",
      "episode:1903 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 82us/step - loss: 0.0433\n",
      "episode:1903 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 82us/step - loss: 0.0438\n",
      "episode:1903 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.0385\n",
      "episode:1903 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 87us/step - loss: 0.0602\n",
      "episode:1903 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 87us/step - loss: 0.4841\n",
      "episode:1903 step:15\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 79us/step - loss: 0.4712\n",
      "episode:1903 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 1.0417\n",
      "episode:1903 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.0544\n",
      "episode:1903 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.4966\n",
      "episode:1903 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0901\n",
      "episode:1903 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0900\n",
      "episode:1903 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 80us/step - loss: 0.0702\n",
      "episode:1903 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.9877\n",
      "episode:1903 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.0418\n",
      "episode:1903 step:24\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.1007\n",
      "episode:1903 step:25\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0226\n",
      "episode:1903 step:26\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 73us/step - loss: 0.0124\n",
      "episode:1903 step:27\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0515\n",
      "episode:1903 step:28\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0553\n",
      "episode:1903 step:29\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 73us/step - loss: 0.0355\n",
      "episode:1903 step:30\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0267\n",
      "episode:1903 step:31\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.0179\n",
      "episode:1903 step:32\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0583\n",
      "episode:1903 step:33\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.1807\n",
      "episode:1903 step:34\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0495\n",
      "episode:1903 step:35\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.1784\n",
      "episode:1903 step:36\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 80us/step - loss: 0.0282\n",
      "episode:1903 step:37\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.3887\n",
      "episode:1903 step:38\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0552\n",
      "episode:1903 step:39\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0332\n",
      "episode:1903 step:40\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.0776\n",
      "episode:1903 step:41\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.0174\n",
      "episode:1903 step:42\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0351\n",
      "episode:1903 step:43\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.0483\n",
      "episode:1903 step:44\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0254\n",
      "episode:1903 step:45\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0488\n",
      "episode:1903 step:46\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 73us/step - loss: 0.0328\n",
      "episode:1903 step:47\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.0420\n",
      "episode:1903 step:48\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 79us/step - loss: 0.0204\n",
      "episode:1903 step:49\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0536\n",
      "episode:1903 step:50\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 73us/step - loss: 0.0991\n",
      "episode:1903 step:51\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.1546\n",
      "episode:1903 step:52\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.0199\n",
      "episode:1903 step:53\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.1970\n",
      "episode:1903 step:54\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.0516\n",
      "episode:1903 step:55\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.0202\n",
      "episode:1903 step:56\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0294\n",
      "episode:1903 step:57\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.0283\n",
      "episode:1903 step:58\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.0452\n",
      "episode:1903 step:59\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.0904\n",
      "episode:1903 step:60\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.2242\n",
      "episode:1903 step:61\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 79us/step - loss: 0.0749\n",
      "episode:1903 step:62\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.0218\n",
      "episode:1903 step:63\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 80us/step - loss: 0.0253\n",
      "episode:1903 step:64\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0395\n",
      "episode:1903 step:65\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0359\n",
      "episode:1903 step:66\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 79us/step - loss: 0.0485\n",
      "total_loss:0 total_reward:-3.4000000000000075\n",
      "episode:1904 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.5802\n",
      "episode:1904 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.0399\n",
      "episode:1904 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0650\n",
      "episode:1904 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.0575\n",
      "episode:1904 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.5454\n",
      "episode:1904 step:5\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32/32 [==============================] - 0s 122us/step - loss: 0.0725\n",
      "episode:1904 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.4663\n",
      "episode:1904 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.5393\n",
      "episode:1904 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 90us/step - loss: 0.0156\n",
      "episode:1904 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.0127\n",
      "episode:1904 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0403\n",
      "episode:1904 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.0157\n",
      "episode:1904 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.0356\n",
      "episode:1904 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.3254\n",
      "episode:1904 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.2349\n",
      "episode:1904 step:15\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.8436\n",
      "episode:1904 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.0258\n",
      "episode:1904 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.4526\n",
      "episode:1904 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.0899\n",
      "episode:1904 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.0865\n",
      "episode:1904 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 79us/step - loss: 0.0133\n",
      "episode:1904 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.0616\n",
      "episode:1904 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 83us/step - loss: 0.0462\n",
      "episode:1904 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.1870\n",
      "episode:1904 step:24\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.0106\n",
      "episode:1904 step:25\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 73us/step - loss: 0.1218\n",
      "episode:1904 step:26\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0399\n",
      "episode:1904 step:27\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.1514\n",
      "episode:1904 step:28\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.0242\n",
      "episode:1904 step:29\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0629\n",
      "episode:1904 step:30\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.0203\n",
      "episode:1904 step:31\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.0718\n",
      "episode:1904 step:32\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 85us/step - loss: 0.0812\n",
      "episode:1904 step:33\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.0751\n",
      "episode:1904 step:34\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 73us/step - loss: 0.1074\n",
      "episode:1904 step:35\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.0098\n",
      "episode:1904 step:36\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0367\n",
      "episode:1904 step:37\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.2697\n",
      "episode:1904 step:38\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 73us/step - loss: 0.0454\n",
      "episode:1904 step:39\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0265\n",
      "episode:1904 step:40\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0388\n",
      "episode:1904 step:41\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 79us/step - loss: 0.0407\n",
      "episode:1904 step:42\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.0697\n",
      "episode:1904 step:43\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0378\n",
      "episode:1904 step:44\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.1071\n",
      "episode:1904 step:45\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.0291\n",
      "episode:1904 step:46\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.1078\n",
      "episode:1904 step:47\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.0253\n",
      "episode:1904 step:48\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 80us/step - loss: 0.0264\n",
      "episode:1904 step:49\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.0148\n",
      "episode:1904 step:50\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.0374\n",
      "episode:1904 step:51\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.1112\n",
      "episode:1904 step:52\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0229\n",
      "episode:1904 step:53\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0158\n",
      "episode:1904 step:54\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.0118\n",
      "episode:1904 step:55\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.1099\n",
      "episode:1904 step:56\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.2552\n",
      "episode:1904 step:57\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.5607\n",
      "episode:1904 step:58\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0095\n",
      "episode:1904 step:59\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0373\n",
      "episode:1904 step:60\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.0590\n",
      "episode:1904 step:61\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.1875\n",
      "episode:1904 step:62\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.1835\n",
      "episode:1904 step:63\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0246\n",
      "episode:1904 step:64\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0702\n",
      "episode:1904 step:65\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.3019\n",
      "episode:1904 step:66\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.1395\n",
      "episode:1904 step:67\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.1768\n",
      "episode:1904 step:68\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.0132\n",
      "episode:1904 step:69\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0121\n",
      "episode:1904 step:70\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 73us/step - loss: 0.0631\n",
      "episode:1904 step:71\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.0468\n",
      "episode:1904 step:72\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 80us/step - loss: 0.2248\n",
      "episode:1904 step:73\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 80us/step - loss: 0.0296\n",
      "episode:1904 step:74\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.0230\n",
      "episode:1904 step:75\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.0260\n",
      "episode:1904 step:76\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.3564\n",
      "episode:1904 step:77\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.0099\n",
      "episode:1904 step:78\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.3731\n",
      "episode:1904 step:79\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.0490\n",
      "episode:1904 step:80\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.1005\n",
      "episode:1904 step:81\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.0636\n",
      "episode:1904 step:82\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.2556\n",
      "episode:1904 step:83\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 79us/step - loss: 0.0295\n",
      "episode:1904 step:84\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.2271\n",
      "episode:1904 step:85\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0305\n",
      "episode:1904 step:86\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.1743\n",
      "episode:1904 step:87\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32/32 [==============================] - 0s 68us/step - loss: 0.2433\n",
      "episode:1904 step:88\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 73us/step - loss: 0.0199\n",
      "episode:1904 step:89\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.0071\n",
      "episode:1904 step:90\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 131us/step - loss: 0.0536\n",
      "episode:1904 step:91\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 73us/step - loss: 0.1165\n",
      "episode:1904 step:92\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 73us/step - loss: 0.0310\n",
      "episode:1904 step:93\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0320\n",
      "episode:1904 step:94\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.1782\n",
      "episode:1904 step:95\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.0553\n",
      "episode:1904 step:96\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.0290\n",
      "episode:1904 step:97\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 79us/step - loss: 0.5696\n",
      "episode:1904 step:98\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0457\n",
      "episode:1904 step:99\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 73us/step - loss: 0.1137\n",
      "episode:1904 step:100\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.0468\n",
      "episode:1904 step:101\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.0468\n",
      "episode:1904 step:102\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.2005\n",
      "episode:1904 step:103\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.0238\n",
      "episode:1904 step:104\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.5364\n",
      "episode:1904 step:105\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0598\n",
      "episode:1904 step:106\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 1.0630\n",
      "episode:1904 step:107\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.3603\n",
      "episode:1904 step:108\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0651\n",
      "episode:1904 step:109\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.0303\n",
      "episode:1904 step:110\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 73us/step - loss: 0.0842\n",
      "episode:1904 step:111\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0353\n",
      "episode:1904 step:112\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.0554\n",
      "episode:1904 step:113\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.7773\n",
      "episode:1904 step:114\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 80us/step - loss: 0.0257\n",
      "episode:1904 step:115\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.0457\n",
      "episode:1904 step:116\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0277\n",
      "episode:1904 step:117\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.0483\n",
      "episode:1904 step:118\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0236\n",
      "episode:1904 step:119\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0297\n",
      "episode:1904 step:120\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.0614\n",
      "episode:1904 step:121\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.0265\n",
      "episode:1904 step:122\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.0529\n",
      "episode:1904 step:123\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.0216\n",
      "episode:1904 step:124\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.0515\n",
      "episode:1904 step:125\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.3487\n",
      "episode:1904 step:126\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 78us/step - loss: 0.0252\n",
      "episode:1904 step:127\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 73us/step - loss: 0.0140\n",
      "episode:1904 step:128\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.3226\n",
      "episode:1904 step:129\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0568\n",
      "episode:1904 step:130\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 79us/step - loss: 0.0143\n",
      "episode:1904 step:131\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0243\n",
      "episode:1904 step:132\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.5394\n",
      "episode:1904 step:133\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0690\n",
      "episode:1904 step:134\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.4212\n",
      "episode:1904 step:135\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.0384\n",
      "episode:1904 step:136\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 76us/step - loss: 0.0303\n",
      "episode:1904 step:137\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 81us/step - loss: 0.4018\n",
      "episode:1904 step:138\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0088\n",
      "episode:1904 step:139\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.1007\n",
      "episode:1904 step:140\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.0853\n",
      "episode:1904 step:141\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 88us/step - loss: 0.0079\n",
      "episode:1904 step:142\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0298\n",
      "episode:1904 step:143\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0078\n",
      "episode:1904 step:144\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0484\n",
      "episode:1904 step:145\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0352\n",
      "episode:1904 step:146\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0128\n",
      "episode:1904 step:147\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.4265\n",
      "episode:1904 step:148\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0833\n",
      "episode:1904 step:149\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0181\n",
      "episode:1904 step:150\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.6190\n",
      "episode:1904 step:151\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0633\n",
      "episode:1904 step:152\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0368\n",
      "episode:1904 step:153\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0118\n",
      "episode:1904 step:154\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0341\n",
      "episode:1904 step:155\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0670\n",
      "episode:1904 step:156\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0383\n",
      "episode:1904 step:157\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.5642\n",
      "episode:1904 step:158\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0846\n",
      "episode:1904 step:159\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.8517\n",
      "episode:1904 step:160\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.3490\n",
      "episode:1904 step:161\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0223\n",
      "episode:1904 step:162\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0431\n",
      "episode:1904 step:163\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.1117\n",
      "episode:1904 step:164\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.9819\n",
      "episode:1904 step:165\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0181\n",
      "episode:1904 step:166\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0457\n",
      "episode:1904 step:167\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0208\n",
      "episode:1904 step:168\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0468\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:1904 step:169\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.5807\n",
      "episode:1904 step:170\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0820\n",
      "episode:1904 step:171\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0791\n",
      "episode:1904 step:172\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0744\n",
      "episode:1904 step:173\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0287\n",
      "episode:1904 step:174\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0442\n",
      "episode:1904 step:175\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0315\n",
      "episode:1904 step:176\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.3519\n",
      "episode:1904 step:177\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0239\n",
      "episode:1904 step:178\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0280\n",
      "episode:1904 step:179\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0377\n",
      "episode:1904 step:180\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0464\n",
      "episode:1904 step:181\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0486\n",
      "episode:1904 step:182\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 1.0641\n",
      "episode:1904 step:183\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0264\n",
      "episode:1904 step:184\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0348\n",
      "episode:1904 step:185\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.1369\n",
      "episode:1904 step:186\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0652\n",
      "total_loss:0 total_reward:8.599999999999994\n",
      "episode:1905 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0935\n",
      "episode:1905 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0427\n",
      "episode:1905 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0478\n",
      "episode:1905 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0108\n",
      "episode:1905 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.6050\n",
      "episode:1905 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 3.6690\n",
      "episode:1905 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0373\n",
      "episode:1905 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0300\n",
      "episode:1905 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.9020\n",
      "episode:1905 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0498\n",
      "episode:1905 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.3030\n",
      "episode:1905 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.1191\n",
      "episode:1905 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.3197\n",
      "episode:1905 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0487\n",
      "episode:1905 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0314\n",
      "episode:1905 step:15\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0425\n",
      "episode:1905 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.1305\n",
      "episode:1905 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0252\n",
      "episode:1905 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0509\n",
      "episode:1905 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0444\n",
      "episode:1905 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.2215\n",
      "episode:1905 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0743\n",
      "episode:1905 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.1271\n",
      "episode:1905 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0262\n",
      "episode:1905 step:24\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0338\n",
      "episode:1905 step:25\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.1718\n",
      "episode:1905 step:26\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.2106\n",
      "episode:1905 step:27\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0369\n",
      "episode:1905 step:28\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0517\n",
      "episode:1905 step:29\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0510\n",
      "episode:1905 step:30\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0496\n",
      "episode:1905 step:31\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.1098\n",
      "episode:1905 step:32\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.7930\n",
      "episode:1905 step:33\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.1544\n",
      "episode:1905 step:34\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.2810\n",
      "episode:1905 step:35\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.4414\n",
      "episode:1905 step:36\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.7705\n",
      "episode:1905 step:37\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0508\n",
      "episode:1905 step:38\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0381\n",
      "episode:1905 step:39\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0095\n",
      "episode:1905 step:40\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0396\n",
      "episode:1905 step:41\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.2273\n",
      "episode:1905 step:42\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0654\n",
      "episode:1905 step:43\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0693\n",
      "episode:1905 step:44\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0211\n",
      "episode:1905 step:45\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.1500\n",
      "episode:1905 step:46\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.3738\n",
      "episode:1905 step:47\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.1121\n",
      "episode:1905 step:48\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.1718\n",
      "episode:1905 step:49\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.2717\n",
      "episode:1905 step:50\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0421\n",
      "episode:1905 step:51\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0663\n",
      "episode:1905 step:52\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0456\n",
      "episode:1905 step:53\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0200\n",
      "episode:1905 step:54\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0142\n",
      "episode:1905 step:55\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0245\n",
      "episode:1905 step:56\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0848\n",
      "episode:1905 step:57\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 1.3558\n",
      "episode:1905 step:58\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0511\n",
      "episode:1905 step:59\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0627\n",
      "episode:1905 step:60\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0176\n",
      "episode:1905 step:61\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0339\n",
      "episode:1905 step:62\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.1848\n",
      "episode:1905 step:63\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.1524\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_loss:0 total_reward:-3.7000000000000064\n",
      "episode:1906 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 1.3254\n",
      "episode:1906 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0530\n",
      "episode:1906 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0335\n",
      "episode:1906 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0589\n",
      "episode:1906 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0524\n",
      "episode:1906 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0565\n",
      "episode:1906 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0813\n",
      "episode:1906 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0456\n",
      "episode:1906 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0150\n",
      "episode:1906 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0341\n",
      "episode:1906 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0302\n",
      "episode:1906 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0868\n",
      "episode:1906 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.1377\n",
      "episode:1906 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.5050\n",
      "episode:1906 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.0210\n",
      "episode:1906 step:15\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0262\n",
      "episode:1906 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.1129\n",
      "episode:1906 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.1035\n",
      "episode:1906 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.4622\n",
      "episode:1906 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0617\n",
      "episode:1906 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0425\n",
      "episode:1906 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0484\n",
      "episode:1906 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0718\n",
      "episode:1906 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0569\n",
      "episode:1906 step:24\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0253\n",
      "episode:1906 step:25\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0373\n",
      "episode:1906 step:26\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0537\n",
      "episode:1906 step:27\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0720\n",
      "episode:1906 step:28\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.1747\n",
      "episode:1906 step:29\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0235\n",
      "episode:1906 step:30\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.2293\n",
      "episode:1906 step:31\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0496\n",
      "episode:1906 step:32\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0642\n",
      "episode:1906 step:33\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.1355\n",
      "episode:1906 step:34\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0320\n",
      "episode:1906 step:35\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.1549\n",
      "episode:1906 step:36\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0133\n",
      "episode:1906 step:37\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.1477\n",
      "episode:1906 step:38\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.3578\n",
      "episode:1906 step:39\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.2021\n",
      "episode:1906 step:40\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.5195\n",
      "episode:1906 step:41\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0984\n",
      "episode:1906 step:42\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0583\n",
      "episode:1906 step:43\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0688\n",
      "episode:1906 step:44\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0196\n",
      "episode:1906 step:45\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0669\n",
      "episode:1906 step:46\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0285\n",
      "episode:1906 step:47\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.3784\n",
      "episode:1906 step:48\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0461\n",
      "episode:1906 step:49\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0264\n",
      "episode:1906 step:50\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0420\n",
      "episode:1906 step:51\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0413\n",
      "episode:1906 step:52\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.2138\n",
      "episode:1906 step:53\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1421\n",
      "episode:1906 step:54\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0286\n",
      "episode:1906 step:55\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.1664\n",
      "episode:1906 step:56\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.1077\n",
      "episode:1906 step:57\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0592\n",
      "episode:1906 step:58\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.4828\n",
      "episode:1906 step:59\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0115\n",
      "episode:1906 step:60\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0438\n",
      "episode:1906 step:61\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0626\n",
      "episode:1906 step:62\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0555\n",
      "episode:1906 step:63\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0269\n",
      "episode:1906 step:64\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0576\n",
      "episode:1906 step:65\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0597\n",
      "total_loss:0 total_reward:-3.500000000000007\n",
      "episode:1907 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0269\n",
      "episode:1907 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.2083\n",
      "episode:1907 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.2474\n",
      "episode:1907 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.1406\n",
      "episode:1907 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.3319\n",
      "episode:1907 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.6033\n",
      "episode:1907 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.2244\n",
      "episode:1907 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.1108\n",
      "episode:1907 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.4266\n",
      "episode:1907 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.1446\n",
      "episode:1907 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1495\n",
      "episode:1907 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0241\n",
      "episode:1907 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0613\n",
      "episode:1907 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0190\n",
      "episode:1907 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.8672\n",
      "episode:1907 step:15\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32/32 [==============================] - 0s 60us/step - loss: 0.0468\n",
      "episode:1907 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0302\n",
      "episode:1907 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.1002\n",
      "episode:1907 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0819\n",
      "episode:1907 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0524\n",
      "episode:1907 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0558\n",
      "episode:1907 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.1006\n",
      "episode:1907 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0219\n",
      "total_loss:0 total_reward:-7.799999999999999\n",
      "episode:1908 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0874\n",
      "episode:1908 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.9021\n",
      "episode:1908 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0358\n",
      "episode:1908 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0115\n",
      "episode:1908 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0618\n",
      "episode:1908 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0805\n",
      "episode:1908 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0865\n",
      "episode:1908 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.6454\n",
      "episode:1908 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.1719\n",
      "episode:1908 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0708\n",
      "episode:1908 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0356\n",
      "episode:1908 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.3029\n",
      "episode:1908 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0464\n",
      "episode:1908 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0227\n",
      "episode:1908 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0371\n",
      "episode:1908 step:15\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0286\n",
      "episode:1908 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0488\n",
      "episode:1908 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0902\n",
      "episode:1908 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0939\n",
      "episode:1908 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.2076\n",
      "episode:1908 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.1730\n",
      "episode:1908 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.1427\n",
      "episode:1908 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0615\n",
      "episode:1908 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.1385\n",
      "episode:1908 step:24\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0314\n",
      "episode:1908 step:25\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0983\n",
      "episode:1908 step:26\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0278\n",
      "episode:1908 step:27\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.1244\n",
      "episode:1908 step:28\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0908\n",
      "episode:1908 step:29\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0522\n",
      "episode:1908 step:30\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0449\n",
      "episode:1908 step:31\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.1508\n",
      "episode:1908 step:32\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0324\n",
      "episode:1908 step:33\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0219\n",
      "episode:1908 step:34\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0434\n",
      "episode:1908 step:35\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0555\n",
      "episode:1908 step:36\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0268\n",
      "episode:1908 step:37\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0278\n",
      "episode:1908 step:38\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.0160\n",
      "episode:1908 step:39\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0167\n",
      "episode:1908 step:40\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0409\n",
      "episode:1908 step:41\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0177\n",
      "episode:1908 step:42\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.2253\n",
      "episode:1908 step:43\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0608\n",
      "episode:1908 step:44\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0296\n",
      "episode:1908 step:45\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0340\n",
      "episode:1908 step:46\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0244\n",
      "episode:1908 step:47\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1036\n",
      "episode:1908 step:48\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0410\n",
      "episode:1908 step:49\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.6800\n",
      "episode:1908 step:50\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0886\n",
      "episode:1908 step:51\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0160\n",
      "episode:1908 step:52\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0464\n",
      "episode:1908 step:53\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0678\n",
      "episode:1908 step:54\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0149\n",
      "episode:1908 step:55\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0487\n",
      "episode:1908 step:56\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.1264\n",
      "episode:1908 step:57\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.8978\n",
      "episode:1908 step:58\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.2411\n",
      "episode:1908 step:59\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0362\n",
      "episode:1908 step:60\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0989\n",
      "episode:1908 step:61\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0280\n",
      "episode:1908 step:62\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0266\n",
      "episode:1908 step:63\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0160\n",
      "episode:1908 step:64\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0366\n",
      "episode:1908 step:65\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.4921\n",
      "episode:1908 step:66\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0329\n",
      "episode:1908 step:67\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0320\n",
      "episode:1908 step:68\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.2451\n",
      "episode:1908 step:69\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0144\n",
      "episode:1908 step:70\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0586\n",
      "episode:1908 step:71\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.2502\n",
      "episode:1908 step:72\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.2968\n",
      "episode:1908 step:73\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0410\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:1908 step:74\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0489\n",
      "episode:1908 step:75\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.1933\n",
      "episode:1908 step:76\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0314\n",
      "episode:1908 step:77\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.1285\n",
      "episode:1908 step:78\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0996\n",
      "episode:1908 step:79\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.1351\n",
      "episode:1908 step:80\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0234\n",
      "episode:1908 step:81\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0439\n",
      "episode:1908 step:82\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.2096\n",
      "episode:1908 step:83\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.2007\n",
      "episode:1908 step:84\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0741\n",
      "episode:1908 step:85\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0377\n",
      "episode:1908 step:86\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0243\n",
      "episode:1908 step:87\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0700\n",
      "episode:1908 step:88\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 94us/step - loss: 0.0486\n",
      "episode:1908 step:89\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.0332\n",
      "episode:1908 step:90\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.1038\n",
      "episode:1908 step:91\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.2531\n",
      "episode:1908 step:92\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0622\n",
      "episode:1908 step:93\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.2261\n",
      "episode:1908 step:94\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0605\n",
      "episode:1908 step:95\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0140\n",
      "episode:1908 step:96\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0404\n",
      "episode:1908 step:97\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0520\n",
      "episode:1908 step:98\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.2337\n",
      "episode:1908 step:99\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.2181\n",
      "episode:1908 step:100\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.1568\n",
      "episode:1908 step:101\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0594\n",
      "episode:1908 step:102\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0461\n",
      "episode:1908 step:103\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.6617\n",
      "episode:1908 step:104\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0653\n",
      "episode:1908 step:105\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0257\n",
      "episode:1908 step:106\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.5303\n",
      "episode:1908 step:107\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.2199\n",
      "episode:1908 step:108\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0762\n",
      "episode:1908 step:109\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.3029\n",
      "episode:1908 step:110\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.1515\n",
      "episode:1908 step:111\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.1780\n",
      "episode:1908 step:112\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.2466\n",
      "episode:1908 step:113\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0368\n",
      "episode:1908 step:114\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.1651\n",
      "episode:1908 step:115\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0154\n",
      "episode:1908 step:116\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0711\n",
      "episode:1908 step:117\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0702\n",
      "episode:1908 step:118\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0462\n",
      "episode:1908 step:119\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.2364\n",
      "episode:1908 step:120\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1037\n",
      "episode:1908 step:121\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0588\n",
      "episode:1908 step:122\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0498\n",
      "episode:1908 step:123\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0319\n",
      "episode:1908 step:124\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0247\n",
      "episode:1908 step:125\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0227\n",
      "episode:1908 step:126\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0297\n",
      "episode:1908 step:127\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.5823\n",
      "episode:1908 step:128\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.1468\n",
      "episode:1908 step:129\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.1765\n",
      "episode:1908 step:130\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.5912\n",
      "episode:1908 step:131\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0268\n",
      "episode:1908 step:132\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0302\n",
      "episode:1908 step:133\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.1090\n",
      "episode:1908 step:134\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.1403\n",
      "episode:1908 step:135\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0692\n",
      "episode:1908 step:136\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0155\n",
      "episode:1908 step:137\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0095\n",
      "episode:1908 step:138\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0102\n",
      "episode:1908 step:139\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0875\n",
      "episode:1908 step:140\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.1481\n",
      "episode:1908 step:141\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0526\n",
      "episode:1908 step:142\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.5384\n",
      "episode:1908 step:143\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0241\n",
      "episode:1908 step:144\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0519\n",
      "episode:1908 step:145\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.1471\n",
      "episode:1908 step:146\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.4806\n",
      "episode:1908 step:147\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 73us/step - loss: 0.0472\n",
      "episode:1908 step:148\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0891\n",
      "episode:1908 step:149\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.3293\n",
      "episode:1908 step:150\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1431\n",
      "episode:1908 step:151\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.1014\n",
      "episode:1908 step:152\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0580\n",
      "episode:1908 step:153\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0306\n",
      "episode:1908 step:154\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0338\n",
      "episode:1908 step:155\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.2542\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:1908 step:156\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0184\n",
      "episode:1908 step:157\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.2762\n",
      "episode:1908 step:158\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0213\n",
      "episode:1908 step:159\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0297\n",
      "episode:1908 step:160\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0220\n",
      "episode:1908 step:161\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.6713\n",
      "episode:1908 step:162\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0547\n",
      "episode:1908 step:163\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.1064\n",
      "episode:1908 step:164\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0481\n",
      "episode:1908 step:165\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.1195\n",
      "episode:1908 step:166\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0352\n",
      "episode:1908 step:167\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.4282\n",
      "episode:1908 step:168\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0058\n",
      "episode:1908 step:169\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.2039\n",
      "episode:1908 step:170\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0409\n",
      "episode:1908 step:171\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0638\n",
      "episode:1908 step:172\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0781\n",
      "episode:1908 step:173\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0640\n",
      "episode:1908 step:174\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0453\n",
      "episode:1908 step:175\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.1043\n",
      "episode:1908 step:176\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0569\n",
      "episode:1908 step:177\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.2681\n",
      "episode:1908 step:178\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0332\n",
      "episode:1908 step:179\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.1257\n",
      "episode:1908 step:180\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0418\n",
      "episode:1908 step:181\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0665\n",
      "episode:1908 step:182\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.8986\n",
      "episode:1908 step:183\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0095\n",
      "episode:1908 step:184\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0817\n",
      "episode:1908 step:185\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.0773\n",
      "episode:1908 step:186\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0531\n",
      "episode:1908 step:187\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0319\n",
      "episode:1908 step:188\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0497\n",
      "episode:1908 step:189\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.0731\n",
      "episode:1908 step:190\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0802\n",
      "episode:1908 step:191\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0635\n",
      "episode:1908 step:192\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0930\n",
      "episode:1908 step:193\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.1936\n",
      "episode:1908 step:194\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.1680\n",
      "episode:1908 step:195\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.9070\n",
      "episode:1908 step:196\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.2937\n",
      "episode:1908 step:197\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0561\n",
      "episode:1908 step:198\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 2.1258\n",
      "episode:1908 step:199\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0620\n",
      "episode:1908 step:200\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.2983\n",
      "episode:1908 step:201\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.4503\n",
      "episode:1908 step:202\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0423\n",
      "episode:1908 step:203\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0432\n",
      "episode:1908 step:204\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.1147\n",
      "episode:1908 step:205\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0700\n",
      "episode:1908 step:206\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0286\n",
      "episode:1908 step:207\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.3103\n",
      "episode:1908 step:208\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.3282\n",
      "episode:1908 step:209\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0311\n",
      "episode:1908 step:210\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.1687\n",
      "episode:1908 step:211\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0891\n",
      "episode:1908 step:212\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0193\n",
      "episode:1908 step:213\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0210\n",
      "total_loss:0 total_reward:11.300000000000033\n",
      "episode:1909 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.1827\n",
      "episode:1909 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0515\n",
      "episode:1909 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0207\n",
      "episode:1909 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0142\n",
      "episode:1909 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0494\n",
      "episode:1909 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0919\n",
      "episode:1909 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0579\n",
      "episode:1909 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0461\n",
      "episode:1909 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0345\n",
      "episode:1909 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0731\n",
      "episode:1909 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.4743\n",
      "episode:1909 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0594\n",
      "episode:1909 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0088\n",
      "episode:1909 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0522\n",
      "episode:1909 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0461\n",
      "episode:1909 step:15\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0291\n",
      "episode:1909 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0423\n",
      "episode:1909 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0215\n",
      "episode:1909 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0256\n",
      "episode:1909 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 98us/step - loss: 0.0290\n",
      "episode:1909 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.8667\n",
      "episode:1909 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.9832\n",
      "episode:1909 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.3350\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:1909 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.4723\n",
      "episode:1909 step:24\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0503\n",
      "episode:1909 step:25\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0599\n",
      "episode:1909 step:26\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0288\n",
      "episode:1909 step:27\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0865\n",
      "episode:1909 step:28\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0096\n",
      "episode:1909 step:29\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.4448\n",
      "episode:1909 step:30\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0537\n",
      "episode:1909 step:31\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0225\n",
      "episode:1909 step:32\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.2367\n",
      "episode:1909 step:33\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.3896\n",
      "episode:1909 step:34\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0806\n",
      "episode:1909 step:35\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0580\n",
      "episode:1909 step:36\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0616\n",
      "episode:1909 step:37\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0636\n",
      "episode:1909 step:38\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.1005\n",
      "episode:1909 step:39\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 1.2774\n",
      "episode:1909 step:40\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0455\n",
      "episode:1909 step:41\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.3674\n",
      "episode:1909 step:42\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0953\n",
      "episode:1909 step:43\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.1286\n",
      "episode:1909 step:44\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0505\n",
      "episode:1909 step:45\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0326\n",
      "episode:1909 step:46\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.3807\n",
      "episode:1909 step:47\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0768\n",
      "episode:1909 step:48\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0137\n",
      "episode:1909 step:49\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.1379\n",
      "episode:1909 step:50\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.1204\n",
      "episode:1909 step:51\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0200\n",
      "episode:1909 step:52\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0110\n",
      "episode:1909 step:53\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0408\n",
      "episode:1909 step:54\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.3287\n",
      "episode:1909 step:55\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 1.0834\n",
      "episode:1909 step:56\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0349\n",
      "episode:1909 step:57\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.1009\n",
      "episode:1909 step:58\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0162\n",
      "episode:1909 step:59\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0636\n",
      "episode:1909 step:60\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0196\n",
      "episode:1909 step:61\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.6001\n",
      "episode:1909 step:62\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0684\n",
      "episode:1909 step:63\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1308\n",
      "episode:1909 step:64\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.6892\n",
      "episode:1909 step:65\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0835\n",
      "episode:1909 step:66\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0755\n",
      "episode:1909 step:67\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.2775\n",
      "episode:1909 step:68\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.1086\n",
      "episode:1909 step:69\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0483\n",
      "episode:1909 step:70\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.4252\n",
      "episode:1909 step:71\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0758\n",
      "episode:1909 step:72\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0482\n",
      "episode:1909 step:73\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0347\n",
      "episode:1909 step:74\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0437\n",
      "episode:1909 step:75\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0451\n",
      "episode:1909 step:76\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 1.6024\n",
      "episode:1909 step:77\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 1.5886\n",
      "episode:1909 step:78\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0586\n",
      "episode:1909 step:79\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0564\n",
      "episode:1909 step:80\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.2910\n",
      "episode:1909 step:81\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.1924\n",
      "episode:1909 step:82\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0845\n",
      "episode:1909 step:83\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.0495\n",
      "episode:1909 step:84\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0398\n",
      "episode:1909 step:85\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0231\n",
      "episode:1909 step:86\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0399\n",
      "episode:1909 step:87\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0560\n",
      "episode:1909 step:88\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.2179\n",
      "episode:1909 step:89\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0415\n",
      "episode:1909 step:90\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0590\n",
      "episode:1909 step:91\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0451\n",
      "episode:1909 step:92\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0239\n",
      "episode:1909 step:93\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.0244\n",
      "episode:1909 step:94\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.6333\n",
      "episode:1909 step:95\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.1643\n",
      "episode:1909 step:96\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0251\n",
      "episode:1909 step:97\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.2192\n",
      "episode:1909 step:98\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0421\n",
      "episode:1909 step:99\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0685\n",
      "episode:1909 step:100\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0456\n",
      "episode:1909 step:101\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0649\n",
      "episode:1909 step:102\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0227\n",
      "episode:1909 step:103\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.1328\n",
      "episode:1909 step:104\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 52us/step - loss: 0.0187\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:1909 step:105\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0495\n",
      "episode:1909 step:106\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0088\n",
      "episode:1909 step:107\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0824\n",
      "episode:1909 step:108\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0680\n",
      "episode:1909 step:109\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0629\n",
      "episode:1909 step:110\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.2325\n",
      "episode:1909 step:111\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.5362\n",
      "episode:1909 step:112\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0230\n",
      "episode:1909 step:113\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0151\n",
      "episode:1909 step:114\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0278\n",
      "episode:1909 step:115\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0599\n",
      "episode:1909 step:116\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0466\n",
      "episode:1909 step:117\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.2682\n",
      "episode:1909 step:118\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0469\n",
      "episode:1909 step:119\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0286\n",
      "episode:1909 step:120\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0798\n",
      "episode:1909 step:121\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0729\n",
      "episode:1909 step:122\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0298\n",
      "episode:1909 step:123\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0579\n",
      "episode:1909 step:124\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0299\n",
      "episode:1909 step:125\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0756\n",
      "episode:1909 step:126\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0465\n",
      "episode:1909 step:127\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0243\n",
      "episode:1909 step:128\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0177\n",
      "episode:1909 step:129\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0091\n",
      "episode:1909 step:130\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.1917\n",
      "episode:1909 step:131\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 1.0405\n",
      "episode:1909 step:132\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.0444\n",
      "episode:1909 step:133\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.1084\n",
      "episode:1909 step:134\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.8857\n",
      "episode:1909 step:135\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.1382\n",
      "episode:1909 step:136\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0438\n",
      "episode:1909 step:137\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0134\n",
      "episode:1909 step:138\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0098\n",
      "episode:1909 step:139\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.8262\n",
      "episode:1909 step:140\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0262\n",
      "episode:1909 step:141\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.1622\n",
      "episode:1909 step:142\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0301\n",
      "episode:1909 step:143\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.2314\n",
      "episode:1909 step:144\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0600\n",
      "episode:1909 step:145\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0515\n",
      "episode:1909 step:146\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.0329\n",
      "episode:1909 step:147\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0487\n",
      "episode:1909 step:148\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.5867\n",
      "episode:1909 step:149\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.1334\n",
      "episode:1909 step:150\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0449\n",
      "episode:1909 step:151\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0090\n",
      "episode:1909 step:152\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.1283\n",
      "episode:1909 step:153\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0405\n",
      "episode:1909 step:154\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0194\n",
      "episode:1909 step:155\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0260\n",
      "episode:1909 step:156\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.3328\n",
      "episode:1909 step:157\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 1.4766\n",
      "episode:1909 step:158\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.2545\n",
      "episode:1909 step:159\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.1451\n",
      "episode:1909 step:160\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0120\n",
      "episode:1909 step:161\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0231\n",
      "episode:1909 step:162\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0350\n",
      "episode:1909 step:163\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0472\n",
      "episode:1909 step:164\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.1101\n",
      "episode:1909 step:165\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0452\n",
      "episode:1909 step:166\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0784\n",
      "episode:1909 step:167\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0504\n",
      "episode:1909 step:168\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0292\n",
      "episode:1909 step:169\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0289\n",
      "episode:1909 step:170\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0230\n",
      "episode:1909 step:171\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0494\n",
      "episode:1909 step:172\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.1646\n",
      "episode:1909 step:173\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 1.1522\n",
      "episode:1909 step:174\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.2919\n",
      "episode:1909 step:175\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.7499\n",
      "episode:1909 step:176\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0873\n",
      "episode:1909 step:177\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0436\n",
      "episode:1909 step:178\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0552\n",
      "episode:1909 step:179\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0344\n",
      "episode:1909 step:180\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0376\n",
      "episode:1909 step:181\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.5504\n",
      "episode:1909 step:182\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0293\n",
      "episode:1909 step:183\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0348\n",
      "episode:1909 step:184\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 99us/step - loss: 0.1302\n",
      "episode:1909 step:185\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.1164\n",
      "episode:1909 step:186\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.2096\n",
      "episode:1909 step:187\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0265\n",
      "episode:1909 step:188\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0656\n",
      "episode:1909 step:189\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.1262\n",
      "episode:1909 step:190\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.1048\n",
      "episode:1909 step:191\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.8469\n",
      "episode:1909 step:192\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0940\n",
      "episode:1909 step:193\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.4948\n",
      "episode:1909 step:194\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.2055\n",
      "episode:1909 step:195\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.1302\n",
      "episode:1909 step:196\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.4583\n",
      "episode:1909 step:197\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.3770\n",
      "episode:1909 step:198\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0765\n",
      "episode:1909 step:199\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.4106\n",
      "episode:1909 step:200\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.6356\n",
      "episode:1909 step:201\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.1083\n",
      "episode:1909 step:202\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1712\n",
      "episode:1909 step:203\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0249\n",
      "episode:1909 step:204\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0259\n",
      "episode:1909 step:205\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.0175\n",
      "episode:1909 step:206\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0238\n",
      "episode:1909 step:207\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0595\n",
      "episode:1909 step:208\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0716\n",
      "episode:1909 step:209\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.1242\n",
      "episode:1909 step:210\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.1247\n",
      "episode:1909 step:211\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0172\n",
      "episode:1909 step:212\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.2392\n",
      "episode:1909 step:213\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0317\n",
      "episode:1909 step:214\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0315\n",
      "episode:1909 step:215\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0475\n",
      "episode:1909 step:216\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0428\n",
      "episode:1909 step:217\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.1103\n",
      "episode:1909 step:218\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.2090\n",
      "episode:1909 step:219\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.1109\n",
      "episode:1909 step:220\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0183\n",
      "episode:1909 step:221\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0506\n",
      "episode:1909 step:222\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0292\n",
      "episode:1909 step:223\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0397\n",
      "episode:1909 step:224\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0160\n",
      "episode:1909 step:225\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0272\n",
      "episode:1909 step:226\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0469\n",
      "episode:1909 step:227\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0517\n",
      "episode:1909 step:228\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0136\n",
      "episode:1909 step:229\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.2898\n",
      "episode:1909 step:230\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0416\n",
      "episode:1909 step:231\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0318\n",
      "episode:1909 step:232\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.1025\n",
      "episode:1909 step:233\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.8503\n",
      "episode:1909 step:234\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0366\n",
      "episode:1909 step:235\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.1005\n",
      "episode:1909 step:236\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0140\n",
      "episode:1909 step:237\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.3199\n",
      "episode:1909 step:238\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0753\n",
      "episode:1909 step:239\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0659\n",
      "episode:1909 step:240\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0426\n",
      "episode:1909 step:241\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0404\n",
      "episode:1909 step:242\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.2217\n",
      "episode:1909 step:243\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0214\n",
      "episode:1909 step:244\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.2031\n",
      "episode:1909 step:245\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0600\n",
      "episode:1909 step:246\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0184\n",
      "episode:1909 step:247\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0615\n",
      "episode:1909 step:248\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.1037\n",
      "episode:1909 step:249\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0382\n",
      "episode:1909 step:250\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0584\n",
      "episode:1909 step:251\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0129\n",
      "total_loss:0 total_reward:15.100000000000087\n",
      "episode:1910 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0160\n",
      "episode:1910 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0413\n",
      "episode:1910 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0287\n",
      "episode:1910 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.2381\n",
      "episode:1910 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0584\n",
      "episode:1910 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0346\n",
      "episode:1910 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0948\n",
      "episode:1910 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.2126\n",
      "episode:1910 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0287\n",
      "episode:1910 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.2940\n",
      "episode:1910 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0321\n",
      "episode:1910 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.2180\n",
      "episode:1910 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0379\n",
      "episode:1910 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.1291\n",
      "episode:1910 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0173\n",
      "episode:1910 step:15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0489\n",
      "episode:1910 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0220\n",
      "episode:1910 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0303\n",
      "episode:1910 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0503\n",
      "episode:1910 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.2337\n",
      "episode:1910 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0434\n",
      "episode:1910 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0286\n",
      "episode:1910 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0393\n",
      "episode:1910 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0856\n",
      "episode:1910 step:24\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0135\n",
      "episode:1910 step:25\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.2309\n",
      "episode:1910 step:26\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0516\n",
      "episode:1910 step:27\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.5632\n",
      "episode:1910 step:28\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0878\n",
      "episode:1910 step:29\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0940\n",
      "episode:1910 step:30\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0230\n",
      "episode:1910 step:31\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.1732\n",
      "episode:1910 step:32\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0292\n",
      "episode:1910 step:33\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0509\n",
      "episode:1910 step:34\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0360\n",
      "episode:1910 step:35\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0683\n",
      "episode:1910 step:36\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0231\n",
      "episode:1910 step:37\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0723\n",
      "episode:1910 step:38\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.6235\n",
      "episode:1910 step:39\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0213\n",
      "episode:1910 step:40\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.8282\n",
      "episode:1910 step:41\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.1785\n",
      "episode:1910 step:42\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0552\n",
      "episode:1910 step:43\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.2121\n",
      "episode:1910 step:44\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0265\n",
      "episode:1910 step:45\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.7825\n",
      "episode:1910 step:46\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0694\n",
      "episode:1910 step:47\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0293\n",
      "episode:1910 step:48\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1083\n",
      "episode:1910 step:49\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0775\n",
      "episode:1910 step:50\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0274\n",
      "episode:1910 step:51\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0631\n",
      "episode:1910 step:52\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0384\n",
      "episode:1910 step:53\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0227\n",
      "episode:1910 step:54\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0747\n",
      "episode:1910 step:55\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0113\n",
      "episode:1910 step:56\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0229\n",
      "episode:1910 step:57\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.1352\n",
      "episode:1910 step:58\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0366\n",
      "episode:1910 step:59\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0225\n",
      "episode:1910 step:60\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0269\n",
      "episode:1910 step:61\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0542\n",
      "episode:1910 step:62\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0340\n",
      "total_loss:0 total_reward:-3.800000000000006\n",
      "episode:1911 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0120\n",
      "episode:1911 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0300\n",
      "episode:1911 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.1803\n",
      "episode:1911 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0164\n",
      "episode:1911 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.6641\n",
      "episode:1911 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0439\n",
      "episode:1911 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.6452\n",
      "episode:1911 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0579\n",
      "episode:1911 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0379\n",
      "episode:1911 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0601\n",
      "episode:1911 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.3142\n",
      "episode:1911 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0372\n",
      "episode:1911 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0180\n",
      "episode:1911 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0363\n",
      "episode:1911 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0169\n",
      "episode:1911 step:15\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0615\n",
      "episode:1911 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0340\n",
      "episode:1911 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0239\n",
      "episode:1911 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.3025\n",
      "episode:1911 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.3224\n",
      "episode:1911 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.4116\n",
      "episode:1911 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0659\n",
      "episode:1911 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.7371\n",
      "episode:1911 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.3155\n",
      "episode:1911 step:24\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 137us/step - loss: 0.0426\n",
      "episode:1911 step:25\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0107\n",
      "episode:1911 step:26\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.1049\n",
      "episode:1911 step:27\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0564\n",
      "episode:1911 step:28\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0045\n",
      "episode:1911 step:29\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0521\n",
      "episode:1911 step:30\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0454\n",
      "episode:1911 step:31\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0594\n",
      "episode:1911 step:32\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.1972\n",
      "episode:1911 step:33\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0958\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:1911 step:34\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0301\n",
      "episode:1911 step:35\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0167\n",
      "episode:1911 step:36\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0642\n",
      "episode:1911 step:37\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0600\n",
      "episode:1911 step:38\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0288\n",
      "episode:1911 step:39\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.4321\n",
      "episode:1911 step:40\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0212\n",
      "episode:1911 step:41\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.5195\n",
      "episode:1911 step:42\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0434\n",
      "episode:1911 step:43\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.1659\n",
      "episode:1911 step:44\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0133\n",
      "episode:1911 step:45\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.7048\n",
      "episode:1911 step:46\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0225\n",
      "episode:1911 step:47\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0189\n",
      "episode:1911 step:48\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0876\n",
      "episode:1911 step:49\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.8145\n",
      "episode:1911 step:50\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0196\n",
      "episode:1911 step:51\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0425\n",
      "episode:1911 step:52\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.2007\n",
      "episode:1911 step:53\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.4501\n",
      "episode:1911 step:54\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0244\n",
      "episode:1911 step:55\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0131\n",
      "episode:1911 step:56\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0384\n",
      "episode:1911 step:57\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0963\n",
      "episode:1911 step:58\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0575\n",
      "episode:1911 step:59\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.1077\n",
      "total_loss:0 total_reward:-4.100000000000005\n",
      "episode:1912 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0316\n",
      "episode:1912 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0320\n",
      "episode:1912 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0233\n",
      "episode:1912 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0095\n",
      "episode:1912 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0800\n",
      "episode:1912 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0800\n",
      "episode:1912 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0645\n",
      "episode:1912 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0488\n",
      "episode:1912 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0293\n",
      "episode:1912 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0063\n",
      "episode:1912 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.1599\n",
      "episode:1912 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0135\n",
      "episode:1912 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0431\n",
      "episode:1912 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0476\n",
      "episode:1912 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.4663\n",
      "episode:1912 step:15\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0306\n",
      "episode:1912 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0586\n",
      "episode:1912 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.3359\n",
      "episode:1912 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0143\n",
      "episode:1912 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.2242\n",
      "episode:1912 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0248\n",
      "episode:1912 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0271\n",
      "episode:1912 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0519\n",
      "episode:1912 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.3228\n",
      "episode:1912 step:24\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.2250\n",
      "episode:1912 step:25\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0239\n",
      "episode:1912 step:26\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0340\n",
      "episode:1912 step:27\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.1178\n",
      "episode:1912 step:28\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0915\n",
      "episode:1912 step:29\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0313\n",
      "episode:1912 step:30\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0197\n",
      "episode:1912 step:31\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0934\n",
      "episode:1912 step:32\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0845\n",
      "episode:1912 step:33\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0724\n",
      "episode:1912 step:34\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0902\n",
      "episode:1912 step:35\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.3730\n",
      "episode:1912 step:36\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.2979\n",
      "episode:1912 step:37\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0572\n",
      "episode:1912 step:38\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.3045\n",
      "episode:1912 step:39\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.2593\n",
      "episode:1912 step:40\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.3456\n",
      "episode:1912 step:41\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0570\n",
      "episode:1912 step:42\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0799\n",
      "episode:1912 step:43\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0939\n",
      "episode:1912 step:44\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0257\n",
      "episode:1912 step:45\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0805\n",
      "episode:1912 step:46\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0248\n",
      "episode:1912 step:47\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0614\n",
      "episode:1912 step:48\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1378\n",
      "episode:1912 step:49\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.3157\n",
      "episode:1912 step:50\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 1.1075\n",
      "episode:1912 step:51\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.1701\n",
      "episode:1912 step:52\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0777\n",
      "episode:1912 step:53\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.1617\n",
      "episode:1912 step:54\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.1701\n",
      "episode:1912 step:55\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0979\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:1912 step:56\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0221\n",
      "episode:1912 step:57\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0314\n",
      "episode:1912 step:58\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0394\n",
      "episode:1912 step:59\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.1006\n",
      "episode:1912 step:60\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0510\n",
      "episode:1912 step:61\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0403\n",
      "episode:1912 step:62\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0459\n",
      "episode:1912 step:63\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0351\n",
      "episode:1912 step:64\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0101\n",
      "episode:1912 step:65\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0286\n",
      "episode:1912 step:66\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0214\n",
      "total_loss:0 total_reward:-3.4000000000000075\n",
      "episode:1913 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0192\n",
      "episode:1913 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.2250\n",
      "episode:1913 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0842\n",
      "episode:1913 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0406\n",
      "episode:1913 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0983\n",
      "episode:1913 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.1578\n",
      "episode:1913 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0509\n",
      "episode:1913 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0369\n",
      "episode:1913 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0387\n",
      "episode:1913 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0216\n",
      "episode:1913 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0176\n",
      "episode:1913 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0829\n",
      "episode:1913 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0586\n",
      "episode:1913 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0276\n",
      "episode:1913 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0640\n",
      "episode:1913 step:15\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0218\n",
      "episode:1913 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0126\n",
      "episode:1913 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.1736\n",
      "episode:1913 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0453\n",
      "episode:1913 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0171\n",
      "episode:1913 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0071\n",
      "episode:1913 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0217\n",
      "episode:1913 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0308\n",
      "episode:1913 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0490\n",
      "episode:1913 step:24\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0146\n",
      "episode:1913 step:25\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0214\n",
      "episode:1913 step:26\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0509\n",
      "episode:1913 step:27\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0465\n",
      "episode:1913 step:28\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0236\n",
      "episode:1913 step:29\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0194\n",
      "episode:1913 step:30\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0381\n",
      "episode:1913 step:31\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0299\n",
      "episode:1913 step:32\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.2302\n",
      "episode:1913 step:33\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0622\n",
      "episode:1913 step:34\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.1425\n",
      "episode:1913 step:35\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0176\n",
      "episode:1913 step:36\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0219\n",
      "episode:1913 step:37\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.5924\n",
      "episode:1913 step:38\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0726\n",
      "episode:1913 step:39\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0332\n",
      "episode:1913 step:40\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0406\n",
      "episode:1913 step:41\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0247\n",
      "episode:1913 step:42\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.4988\n",
      "episode:1913 step:43\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0077\n",
      "episode:1913 step:44\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0266\n",
      "episode:1913 step:45\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0343\n",
      "episode:1913 step:46\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.3207\n",
      "episode:1913 step:47\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0643\n",
      "episode:1913 step:48\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.9598\n",
      "episode:1913 step:49\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0422\n",
      "episode:1913 step:50\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0248\n",
      "episode:1913 step:51\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0268\n",
      "episode:1913 step:52\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0633\n",
      "episode:1913 step:53\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.8313\n",
      "episode:1913 step:54\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0312\n",
      "episode:1913 step:55\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0210\n",
      "episode:1913 step:56\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0162\n",
      "episode:1913 step:57\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0450\n",
      "episode:1913 step:58\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0768\n",
      "episode:1913 step:59\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0915\n",
      "episode:1913 step:60\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.1081\n",
      "episode:1913 step:61\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0524\n",
      "episode:1913 step:62\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0751\n",
      "episode:1913 step:63\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0159\n",
      "episode:1913 step:64\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.7863\n",
      "episode:1913 step:65\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0139\n",
      "episode:1913 step:66\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0345\n",
      "episode:1913 step:67\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0095\n",
      "episode:1913 step:68\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0226\n",
      "episode:1913 step:69\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0646\n",
      "episode:1913 step:70\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.1055\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:1913 step:71\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0281\n",
      "episode:1913 step:72\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0131\n",
      "episode:1913 step:73\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.4696\n",
      "episode:1913 step:74\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.0181\n",
      "episode:1913 step:75\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0089\n",
      "episode:1913 step:76\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0178\n",
      "episode:1913 step:77\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0431\n",
      "episode:1913 step:78\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0364\n",
      "episode:1913 step:79\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.1696\n",
      "episode:1913 step:80\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.1655\n",
      "episode:1913 step:81\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0210\n",
      "episode:1913 step:82\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0768\n",
      "episode:1913 step:83\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.2159\n",
      "episode:1913 step:84\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0402\n",
      "episode:1913 step:85\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.1265\n",
      "episode:1913 step:86\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0777\n",
      "episode:1913 step:87\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0245\n",
      "episode:1913 step:88\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.9104\n",
      "episode:1913 step:89\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0696\n",
      "episode:1913 step:90\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.6426\n",
      "episode:1913 step:91\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0264\n",
      "episode:1913 step:92\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0381\n",
      "episode:1913 step:93\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0279\n",
      "episode:1913 step:94\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0623\n",
      "episode:1913 step:95\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.5977\n",
      "episode:1913 step:96\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.3646\n",
      "episode:1913 step:97\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.3966\n",
      "episode:1913 step:98\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 74us/step - loss: 0.0424\n",
      "episode:1913 step:99\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0355\n",
      "episode:1913 step:100\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0190\n",
      "episode:1913 step:101\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0296\n",
      "episode:1913 step:102\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0400\n",
      "episode:1913 step:103\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0916\n",
      "episode:1913 step:104\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.3552\n",
      "episode:1913 step:105\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.2681\n",
      "episode:1913 step:106\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.0301\n",
      "episode:1913 step:107\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0380\n",
      "episode:1913 step:108\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0419\n",
      "episode:1913 step:109\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0254\n",
      "total_loss:0 total_reward:0.8999999999999773\n",
      "episode:1914 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.1400\n",
      "episode:1914 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0076\n",
      "episode:1914 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0771\n",
      "episode:1914 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0142\n",
      "episode:1914 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.2303\n",
      "episode:1914 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0843\n",
      "episode:1914 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0470\n",
      "episode:1914 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0109\n",
      "episode:1914 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.0252\n",
      "episode:1914 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0459\n",
      "episode:1914 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0188\n",
      "episode:1914 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0733\n",
      "episode:1914 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0612\n",
      "episode:1914 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1657\n",
      "episode:1914 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.1664\n",
      "episode:1914 step:15\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.2002\n",
      "episode:1914 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 1.1684\n",
      "episode:1914 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0172\n",
      "episode:1914 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0802\n",
      "episode:1914 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0274\n",
      "episode:1914 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.2034\n",
      "episode:1914 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.4462\n",
      "episode:1914 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.4385\n",
      "episode:1914 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0250\n",
      "episode:1914 step:24\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0121\n",
      "episode:1914 step:25\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0295\n",
      "episode:1914 step:26\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0621\n",
      "episode:1914 step:27\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.4930\n",
      "episode:1914 step:28\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 103us/step - loss: 0.0497\n",
      "episode:1914 step:29\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.1414\n",
      "episode:1914 step:30\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 77us/step - loss: 0.2964\n",
      "episode:1914 step:31\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0440\n",
      "episode:1914 step:32\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0897\n",
      "episode:1914 step:33\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0295\n",
      "episode:1914 step:34\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.4264\n",
      "episode:1914 step:35\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.3416\n",
      "episode:1914 step:36\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.1501\n",
      "episode:1914 step:37\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0186\n",
      "episode:1914 step:38\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0352\n",
      "episode:1914 step:39\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0307\n",
      "episode:1914 step:40\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0452\n",
      "episode:1914 step:41\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.1770\n",
      "episode:1914 step:42\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.2262\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:1914 step:43\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0574\n",
      "episode:1914 step:44\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0985\n",
      "episode:1914 step:45\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0442\n",
      "episode:1914 step:46\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0513\n",
      "episode:1914 step:47\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.2228\n",
      "episode:1914 step:48\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.1025\n",
      "episode:1914 step:49\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0181\n",
      "episode:1914 step:50\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0475\n",
      "episode:1914 step:51\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.1232\n",
      "episode:1914 step:52\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0798\n",
      "episode:1914 step:53\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.3567\n",
      "episode:1914 step:54\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0208\n",
      "episode:1914 step:55\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0148\n",
      "episode:1914 step:56\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.1683\n",
      "episode:1914 step:57\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0132\n",
      "episode:1914 step:58\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.8194\n",
      "episode:1914 step:59\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0412\n",
      "episode:1914 step:60\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1994\n",
      "episode:1914 step:61\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0225\n",
      "episode:1914 step:62\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0300\n",
      "episode:1914 step:63\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0769\n",
      "episode:1914 step:64\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0226\n",
      "episode:1914 step:65\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0179\n",
      "total_loss:0 total_reward:-3.500000000000007\n",
      "episode:1915 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0459\n",
      "episode:1915 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.1529\n",
      "episode:1915 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0678\n",
      "episode:1915 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.3664\n",
      "episode:1915 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.5399\n",
      "episode:1915 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0862\n",
      "episode:1915 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0361\n",
      "episode:1915 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.5875\n",
      "episode:1915 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.8474\n",
      "episode:1915 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0360\n",
      "episode:1915 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0925\n",
      "episode:1915 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.2047\n",
      "episode:1915 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0701\n",
      "episode:1915 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.6304\n",
      "episode:1915 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0456\n",
      "episode:1915 step:15\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0548\n",
      "episode:1915 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0131\n",
      "episode:1915 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0255\n",
      "episode:1915 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0657\n",
      "episode:1915 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0396\n",
      "episode:1915 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0790\n",
      "episode:1915 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0116\n",
      "episode:1915 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0746\n",
      "episode:1915 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0825\n",
      "episode:1915 step:24\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0660\n",
      "episode:1915 step:25\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 75us/step - loss: 0.1057\n",
      "episode:1915 step:26\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0596\n",
      "episode:1915 step:27\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0776\n",
      "episode:1915 step:28\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0491\n",
      "episode:1915 step:29\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0256\n",
      "episode:1915 step:30\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0454\n",
      "episode:1915 step:31\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0264\n",
      "episode:1915 step:32\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0381\n",
      "episode:1915 step:33\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0585\n",
      "episode:1915 step:34\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.2535\n",
      "episode:1915 step:35\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0668\n",
      "episode:1915 step:36\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.3676\n",
      "episode:1915 step:37\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0877\n",
      "episode:1915 step:38\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0236\n",
      "episode:1915 step:39\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0179\n",
      "episode:1915 step:40\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0097\n",
      "episode:1915 step:41\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0274\n",
      "episode:1915 step:42\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0387\n",
      "episode:1915 step:43\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0498\n",
      "episode:1915 step:44\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0137\n",
      "episode:1915 step:45\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.1775\n",
      "episode:1915 step:46\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0192\n",
      "episode:1915 step:47\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.5971\n",
      "episode:1915 step:48\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0234\n",
      "episode:1915 step:49\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.1575\n",
      "episode:1915 step:50\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1585\n",
      "episode:1915 step:51\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1566\n",
      "episode:1915 step:52\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1297\n",
      "episode:1915 step:53\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0308\n",
      "episode:1915 step:54\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0867\n",
      "episode:1915 step:55\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0220\n",
      "episode:1915 step:56\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0354\n",
      "episode:1915 step:57\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0080\n",
      "episode:1915 step:58\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0132\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:1915 step:59\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0376\n",
      "episode:1915 step:60\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0690\n",
      "episode:1915 step:61\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0239\n",
      "episode:1915 step:62\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0213\n",
      "episode:1915 step:63\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0161\n",
      "total_loss:0 total_reward:-3.7000000000000064\n",
      "episode:1916 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.2177\n",
      "episode:1916 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0353\n",
      "episode:1916 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.2538\n",
      "episode:1916 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0595\n",
      "episode:1916 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0088\n",
      "episode:1916 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 1.0480\n",
      "episode:1916 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0075\n",
      "episode:1916 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0814\n",
      "episode:1916 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.3821\n",
      "episode:1916 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0287\n",
      "episode:1916 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0878\n",
      "episode:1916 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0789\n",
      "episode:1916 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0199\n",
      "episode:1916 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.1451\n",
      "episode:1916 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0400\n",
      "episode:1916 step:15\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0685\n",
      "episode:1916 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0393\n",
      "episode:1916 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0449\n",
      "episode:1916 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.1053\n",
      "episode:1916 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0393\n",
      "episode:1916 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0459\n",
      "episode:1916 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0153\n",
      "episode:1916 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.8003\n",
      "episode:1916 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0512\n",
      "episode:1916 step:24\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.4879\n",
      "episode:1916 step:25\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0297\n",
      "episode:1916 step:26\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0287\n",
      "episode:1916 step:27\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0121\n",
      "episode:1916 step:28\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.4792\n",
      "episode:1916 step:29\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.3602\n",
      "episode:1916 step:30\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0578\n",
      "episode:1916 step:31\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.1964\n",
      "episode:1916 step:32\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0136\n",
      "episode:1916 step:33\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0686\n",
      "episode:1916 step:34\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.3909\n",
      "episode:1916 step:35\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0244\n",
      "episode:1916 step:36\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0539\n",
      "episode:1916 step:37\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0439\n",
      "episode:1916 step:38\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0244\n",
      "episode:1916 step:39\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0379\n",
      "episode:1916 step:40\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0520\n",
      "episode:1916 step:41\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.1129\n",
      "episode:1916 step:42\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0109\n",
      "episode:1916 step:43\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0282\n",
      "episode:1916 step:44\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0519\n",
      "episode:1916 step:45\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0238\n",
      "episode:1916 step:46\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0211\n",
      "episode:1916 step:47\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0198\n",
      "episode:1916 step:48\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.2170\n",
      "episode:1916 step:49\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.1313\n",
      "episode:1916 step:50\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0496\n",
      "episode:1916 step:51\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0254\n",
      "episode:1916 step:52\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0887\n",
      "episode:1916 step:53\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0409\n",
      "episode:1916 step:54\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0406\n",
      "episode:1916 step:55\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.1132\n",
      "episode:1916 step:56\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0991\n",
      "episode:1916 step:57\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.1630\n",
      "episode:1916 step:58\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.8307\n",
      "episode:1916 step:59\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.1149\n",
      "episode:1916 step:60\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0657\n",
      "episode:1916 step:61\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0103\n",
      "episode:1916 step:62\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0128\n",
      "episode:1916 step:63\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0605\n",
      "episode:1916 step:64\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0698\n",
      "episode:1916 step:65\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.2607\n",
      "episode:1916 step:66\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0363\n",
      "episode:1916 step:67\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 1.8551\n",
      "episode:1916 step:68\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.3136\n",
      "episode:1916 step:69\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1972\n",
      "episode:1916 step:70\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0675\n",
      "episode:1916 step:71\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0535\n",
      "episode:1916 step:72\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0652\n",
      "episode:1916 step:73\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0240\n",
      "episode:1916 step:74\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0126\n",
      "episode:1916 step:75\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.8536\n",
      "episode:1916 step:76\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0548\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:1916 step:77\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0998\n",
      "episode:1916 step:78\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.2193\n",
      "episode:1916 step:79\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0189\n",
      "episode:1916 step:80\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0195\n",
      "episode:1916 step:81\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0439\n",
      "episode:1916 step:82\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0283\n",
      "episode:1916 step:83\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.1557\n",
      "episode:1916 step:84\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0399\n",
      "episode:1916 step:85\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1219\n",
      "episode:1916 step:86\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0241\n",
      "episode:1916 step:87\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 72us/step - loss: 0.0979\n",
      "episode:1916 step:88\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0751\n",
      "episode:1916 step:89\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0790\n",
      "episode:1916 step:90\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0419\n",
      "episode:1916 step:91\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0132\n",
      "episode:1916 step:92\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0223\n",
      "episode:1916 step:93\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0480\n",
      "episode:1916 step:94\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0327\n",
      "episode:1916 step:95\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0681\n",
      "episode:1916 step:96\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0163\n",
      "episode:1916 step:97\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0749\n",
      "episode:1916 step:98\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0374\n",
      "episode:1916 step:99\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0166\n",
      "episode:1916 step:100\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.1243\n",
      "episode:1916 step:101\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0241\n",
      "episode:1916 step:102\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0194\n",
      "episode:1916 step:103\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0317\n",
      "episode:1916 step:104\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0396\n",
      "episode:1916 step:105\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0669\n",
      "episode:1916 step:106\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0921\n",
      "episode:1916 step:107\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.1225\n",
      "episode:1916 step:108\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0172\n",
      "episode:1916 step:109\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0479\n",
      "episode:1916 step:110\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.8570\n",
      "episode:1916 step:111\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0967\n",
      "episode:1916 step:112\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0284\n",
      "episode:1916 step:113\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0326\n",
      "episode:1916 step:114\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0428\n",
      "episode:1916 step:115\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0124\n",
      "episode:1916 step:116\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0683\n",
      "episode:1916 step:117\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0484\n",
      "episode:1916 step:118\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0177\n",
      "episode:1916 step:119\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0218\n",
      "episode:1916 step:120\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0783\n",
      "episode:1916 step:121\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0794\n",
      "episode:1916 step:122\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.1332\n",
      "episode:1916 step:123\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0196\n",
      "episode:1916 step:124\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0265\n",
      "episode:1916 step:125\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0379\n",
      "episode:1916 step:126\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0559\n",
      "episode:1916 step:127\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.2539\n",
      "episode:1916 step:128\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0110\n",
      "episode:1916 step:129\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.3025\n",
      "episode:1916 step:130\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.2201\n",
      "episode:1916 step:131\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0276\n",
      "episode:1916 step:132\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0857\n",
      "episode:1916 step:133\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0649\n",
      "total_loss:0 total_reward:3.2999999999999687\n",
      "episode:1917 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.4685\n",
      "episode:1917 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0125\n",
      "episode:1917 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 51us/step - loss: 0.1045\n",
      "episode:1917 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0833\n",
      "episode:1917 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0487\n",
      "episode:1917 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.1160\n",
      "episode:1917 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.0411\n",
      "episode:1917 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.1609\n",
      "episode:1917 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.1545\n",
      "episode:1917 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0752\n",
      "episode:1917 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.1246\n",
      "episode:1917 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0492\n",
      "episode:1917 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0643\n",
      "episode:1917 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.5518\n",
      "episode:1917 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.1721\n",
      "episode:1917 step:15\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0233\n",
      "episode:1917 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0224\n",
      "episode:1917 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0089\n",
      "episode:1917 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.1267\n",
      "episode:1917 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0167\n",
      "episode:1917 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0196\n",
      "episode:1917 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.1360\n",
      "episode:1917 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0408\n",
      "episode:1917 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0218\n",
      "episode:1917 step:24\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32/32 [==============================] - 0s 61us/step - loss: 0.4436\n",
      "episode:1917 step:25\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.2334\n",
      "episode:1917 step:26\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0333\n",
      "episode:1917 step:27\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0143\n",
      "episode:1917 step:28\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0266\n",
      "episode:1917 step:29\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0465\n",
      "episode:1917 step:30\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 67us/step - loss: 0.0958\n",
      "episode:1917 step:31\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0353\n",
      "episode:1917 step:32\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0311\n",
      "episode:1917 step:33\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 53us/step - loss: 0.0310\n",
      "episode:1917 step:34\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.1112\n",
      "episode:1917 step:35\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0303\n",
      "episode:1917 step:36\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0333\n",
      "episode:1917 step:37\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.3504\n",
      "episode:1917 step:38\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.1236\n",
      "episode:1917 step:39\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0238\n",
      "episode:1917 step:40\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0595\n",
      "episode:1917 step:41\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.7462\n",
      "episode:1917 step:42\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0801\n",
      "episode:1917 step:43\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0468\n",
      "episode:1917 step:44\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.9119\n",
      "episode:1917 step:45\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.0093\n",
      "episode:1917 step:46\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.2464\n",
      "episode:1917 step:47\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0546\n",
      "episode:1917 step:48\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.1170\n",
      "episode:1917 step:49\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.1281\n",
      "episode:1917 step:50\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0214\n",
      "episode:1917 step:51\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0108\n",
      "episode:1917 step:52\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 68us/step - loss: 0.0418\n",
      "episode:1917 step:53\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 71us/step - loss: 0.2080\n",
      "episode:1917 step:54\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.0333\n",
      "episode:1917 step:55\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 70us/step - loss: 0.0246\n",
      "episode:1917 step:56\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0486\n",
      "episode:1917 step:57\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0184\n",
      "episode:1917 step:58\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0164\n",
      "episode:1917 step:59\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0184\n",
      "episode:1917 step:60\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0366\n",
      "episode:1917 step:61\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.4962\n",
      "episode:1917 step:62\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.2610\n",
      "episode:1917 step:63\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 1.0605\n",
      "episode:1917 step:64\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0494\n",
      "episode:1917 step:65\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0372\n",
      "episode:1917 step:66\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0391\n",
      "total_loss:0 total_reward:-3.4000000000000075\n",
      "episode:1918 step:0\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0188\n",
      "episode:1918 step:1\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0145\n",
      "episode:1918 step:2\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0494\n",
      "episode:1918 step:3\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.1931\n",
      "episode:1918 step:4\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.1669\n",
      "episode:1918 step:5\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.4203\n",
      "episode:1918 step:6\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.0488\n",
      "episode:1918 step:7\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0310\n",
      "episode:1918 step:8\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0386\n",
      "episode:1918 step:9\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0762\n",
      "episode:1918 step:10\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0195\n",
      "episode:1918 step:11\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.1334\n",
      "episode:1918 step:12\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.2068\n",
      "episode:1918 step:13\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.1224\n",
      "episode:1918 step:14\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0908\n",
      "episode:1918 step:15\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.4992\n",
      "episode:1918 step:16\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.1932\n",
      "episode:1918 step:17\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0577\n",
      "episode:1918 step:18\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.0363\n",
      "episode:1918 step:19\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0506\n",
      "episode:1918 step:20\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 97us/step - loss: 0.0960\n",
      "episode:1918 step:21\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 54us/step - loss: 0.2119\n",
      "episode:1918 step:22\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.5895\n",
      "episode:1918 step:23\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 56us/step - loss: 0.6017\n",
      "episode:1918 step:24\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.1094\n",
      "episode:1918 step:25\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 61us/step - loss: 0.0642\n",
      "episode:1918 step:26\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0127\n",
      "episode:1918 step:27\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.2382\n",
      "episode:1918 step:28\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.0930\n",
      "episode:1918 step:29\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0792\n",
      "episode:1918 step:30\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 64us/step - loss: 0.0515\n",
      "episode:1918 step:31\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 57us/step - loss: 0.3078\n",
      "episode:1918 step:32\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0154\n",
      "episode:1918 step:33\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0669\n",
      "episode:1918 step:34\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0355\n",
      "episode:1918 step:35\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 65us/step - loss: 0.0169\n",
      "episode:1918 step:36\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.6840\n",
      "episode:1918 step:37\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0305\n",
      "episode:1918 step:38\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0732\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:1918 step:39\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 1.3378\n",
      "episode:1918 step:40\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0197\n",
      "episode:1918 step:41\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.2389\n",
      "episode:1918 step:42\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0237\n",
      "episode:1918 step:43\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.4697\n",
      "episode:1918 step:44\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.0651\n",
      "episode:1918 step:45\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 63us/step - loss: 0.1003\n",
      "episode:1918 step:46\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 59us/step - loss: 0.0751\n",
      "episode:1918 step:47\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 66us/step - loss: 0.5067\n",
      "episode:1918 step:48\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 69us/step - loss: 0.0135\n",
      "episode:1918 step:49\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 101us/step - loss: 0.1726\n",
      "episode:1918 step:50\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0302\n",
      "episode:1918 step:51\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.0450\n",
      "episode:1918 step:52\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.1295\n",
      "episode:1918 step:53\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 94us/step - loss: 0.0457\n",
      "episode:1918 step:54\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 62us/step - loss: 0.1456\n",
      "episode:1918 step:55\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 58us/step - loss: 0.3066\n",
      "episode:1918 step:56\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 60us/step - loss: 0.0433\n",
      "episode:1918 step:57\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0795\n",
      "episode:1918 step:58\n",
      "Epoch 1/1\n",
      "32/32 [==============================] - 0s 55us/step - loss: 0.0808\n",
      "episode:1918 step:59\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-25-e1d2be446ab8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# env = gym.make(ENV_NAME)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m# agent = DQN(env)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-24-629001aadcf7>\u001b[0m in \u001b[0;36mmain\u001b[0;34m(env)\u001b[0m\n\u001b[1;32m     28\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m             \u001b[0mreward\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m         \u001b[0magent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mperceive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate_shadow\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnext_state_shadow\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     31\u001b[0m \u001b[0;31m#         if cur_loss is not None:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0;31m#             total_loss += cur_loss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-7-448273cfc41c>\u001b[0m in \u001b[0;36mperceive\u001b[0;34m(self, state, action, reward, next_state, done)\u001b[0m\n\u001b[1;32m     52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplay_buffer\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_Q_network\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     55\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mtrain_Q_network\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-7-448273cfc41c>\u001b[0m in \u001b[0;36mtrain_Q_network\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     78\u001b[0m         \u001b[0mlabel_f\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maction_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     79\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 80\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate_batch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlabel_f\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     81\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0megreedy_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/Keras-2.3.1-py3.6.egg/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m   1237\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1238\u001b[0m                                         \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1239\u001b[0;31m                                         validation_freq=validation_freq)\n\u001b[0m\u001b[1;32m   1240\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1241\u001b[0m     def evaluate(self,\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/Keras-2.3.1-py3.6.egg/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, fit_function, fit_inputs, out_labels, batch_size, epochs, verbose, callbacks, val_function, val_inputs, shuffle, initial_epoch, steps_per_epoch, validation_steps, validation_freq)\u001b[0m\n\u001b[1;32m    194\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 196\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfit_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    197\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3726\u001b[0m         \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3727\u001b[0m       \u001b[0mconverted_inputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3728\u001b[0;31m     \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_graph_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mconverted_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3729\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3730\u001b[0m     \u001b[0;31m# EagerTensor.numpy() will often make a copy to ensure memory safety.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1527\u001b[0m       \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mFor\u001b[0m \u001b[0minvalid\u001b[0m \u001b[0mpositional\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mkeyword\u001b[0m \u001b[0margument\u001b[0m \u001b[0mcombinations\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1528\u001b[0m     \"\"\"\n\u001b[0;32m-> 1529\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1530\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1531\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1567\u001b[0m       raise TypeError(\"Keyword arguments {} unknown. Expected {}.\".format(\n\u001b[1;32m   1568\u001b[0m           list(kwargs.keys()), list(self._arg_keywords)))\n\u001b[0;32m-> 1569\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1570\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1571\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1668\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1669\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1670\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1671\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1672\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    519\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"executor_type\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexecutor_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"config_proto\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    522\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tensorflow.TFE_Py_Execute(ctx._handle, device_name,\n\u001b[1;32m     60\u001b[0m                                                \u001b[0mop_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m                                                num_outputs)\n\u001b[0m\u001b[1;32m     62\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "main(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent.model.load_weights(\"models/model2_1000.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid character in identifier (<ipython-input-46-e669f797f058>, line 28)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-46-e669f797f058>\"\u001b[0;36m, line \u001b[0;32m28\u001b[0m\n\u001b[0;31m    ？\u001b[0m\n\u001b[0m    ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid character in identifier\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "from IPython import display\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "total_reward = 0\n",
    "env = gym.make('Breakout-v0')\n",
    "state = env.reset()\n",
    "(x1,y1,p1,flag) = preprocess(state)\n",
    "state_shadow = np.array([x1,y1,x1,y1,p1])\n",
    "for j in range(1000):\n",
    "    env.render(mode='rgb_array')\n",
    "    action = agent.action(state_shadow) # direct action for test\n",
    "    #action = env.action_space.sample()\n",
    "    next_state,reward,done,_ = env.step(action)\n",
    "    #state = preprocess(state)\n",
    "    (x2,y2,p2,flag) = preprocess(next_state)\n",
    "    if flag == 1:\n",
    "        reward = -1\n",
    "        #done = True\n",
    "    else:\n",
    "        reward = 0.1\n",
    "    next_state_shadow = np.array([x1,y1,x2,y2,p2])\n",
    "    total_reward += reward\n",
    "    state_shadow = next_state_shadow\n",
    "    x1,y1,p1 = x2,y2,p2\n",
    "    cv2.putText(next_state, \"%d:%d:%d:%d:%d:%d\" %(action,total_reward,j,x2,y2,p2), (20, 30), cv2.FONT_HERSHEY_PLAIN, 1.0, (0, 0, 255), 0)\n",
    "#     plt.imshow(next_state)\n",
    "#     display.display(plt.gcf())\n",
    "#     display.clear_output(wait=True)\n",
    "    if done:\n",
    "        break\n",
    "print(total_reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-1 -1 24 74 91] 1\n",
      "[24 74 26 76 91] 1\n",
      "[ 26  76  30  80 103] 2\n",
      "[ 30  80  34  84 113] 1\n",
      "[ 34  84  38  88 113] 1\n",
      "[ 38  88  41  91 113] 1\n",
      "[ 41  91  43  93 116] 2\n",
      "[ 43  93  46  96 129] 2\n",
      "[ 46  96  48  98 132] 2\n",
      "[ 48  98  51 101 132] 2\n",
      "[ 51 101  54 104 132] 2\n",
      "[ 54 104  57 107 132] 2\n",
      "[ 57 107  60 110 132] 2\n",
      "[ 60 110  63 113 132] 2\n",
      "[ 63 113  65 115 132] 2\n",
      "[ 65 115  67 117 132] 2\n",
      "[ 67 117  70 120 132] 2\n",
      "[ 70 120  73 123 132] 2\n",
      "[ 73 123  76 126 132] 2\n",
      "[ 76 126  80 130 132] 2\n",
      "[ 80 130  83 133 132] 2\n",
      "[ 83 133  86 136 132] 2\n",
      "[ 86 136  83 131 132] 2\n",
      "[ 83 131  79 125 132] 2\n",
      "[ 79 125  75 119 132] 2\n",
      "[ 75 119  73 116 132] 2\n",
      "[ 73 116  71 113 132] 2\n",
      "[ 71 113  68 109 132] 2\n",
      "[ 68 109  66 106 132] 2\n",
      "[ 66 106  63 101 132] 2\n",
      "[ 63 101  61  98 132] 3\n",
      "[ 61  98  58  94 132] 3\n",
      "[ 58  94  56  91 132] 3\n",
      "[ 56  91  54  88 132] 3\n",
      "[ 54  88  51  83 128] 3\n",
      "[ 51  83  47  77 105] 3\n",
      "[47 77 43 71 83] 3\n",
      "[43 71 41 68 71] 3\n",
      "[41 68 38 64 54] 3\n",
      "[38 64 34 58 31] 3\n",
      "[34 58 31 53 13] 3\n",
      "[31 53 27 47  0] 3\n",
      "[27 47 23 41  0] 3\n",
      "[23 41 21 38  0] 3\n",
      "[21 38 18 34  0] 3\n",
      "[18 34 16 31  0] 3\n",
      "[16 31 13 26  0] 3\n",
      "[13 26 10 22  0] 3\n",
      "[10 22  7 17  0] 3\n",
      "[ 7 17  5 14  0] 3\n",
      "[ 5 14  3 11  0] 3\n",
      "[ 3 11  0  7  0] 3\n",
      "[ 0  7 -1 -1  0] 3\n",
      "[-1 -1 -1 -1  1] 2\n",
      "[-1 -1 -1 -1  7] 1\n",
      "[-1 -1  0 13  7] 1\n",
      "[ 0 13  3 17  7] 1\n",
      "[ 3 17  7 23  0] 3\n",
      "[ 7 23 10 28  0] 1\n",
      "[10 28 12 31  0] 1\n",
      "[12 31 14 34  0] 3\n",
      "[14 34 17 38  0] 3\n",
      "[17 38 19 41  0] 1\n",
      "[19 41 21 44  1] 2\n",
      "[21 44 25 50 20] 2\n",
      "[25 50 29 56 29] 1\n",
      "[29 56 32 61 29] 1\n",
      "[32 61 35 65 29] 2\n",
      "[35 65 38 70 42] 2\n",
      "[38 70 42 76 66] 2\n",
      "[42 76 44 79 73] 1\n",
      "[44 79 48 85 88] 2\n",
      "[48 85 50 88 97] 1\n",
      "[ 50  88  53  92 107] 2\n",
      "[ 53  92  56  97 122] 1\n",
      "[ 56  97  60 103 130] 1\n",
      "[ 60 103  64 109 130] 1\n",
      "[ 64 109  68 115 130] 1\n",
      "[ 68 115  70 118 130] 1\n",
      "[ 70 118  72 121 132] 2\n",
      "[ 72 121  75 125 132] 2\n",
      "[ 75 125  77 128 132] 2\n",
      "[ 77 128  79 131 132] 2\n",
      "[ 79 131  83 137 132] 2\n",
      "[ 83 137  85 140 132] 2\n",
      "[ 85 140  83 136 132] 2\n",
      "[ 83 136  80 134 132] 2\n",
      "[ 80 134  74 130 132] 2\n",
      "[ 74 130  68 126 132] 2\n",
      "[ 68 126  62 122 132] 2\n",
      "[ 62 122  57 119 132] 2\n",
      "[ 57 119  51 115 132] 2\n",
      "[ 51 115  45 111 132] 2\n",
      "[ 45 111  39 107 132] 2\n",
      "[ 39 107  36 105 132] 2\n",
      "[ 36 105  30 101 132] 2\n",
      "[ 30 101  27  99 132] 2\n",
      "[ 27  99  21  95 132] 2\n",
      "[ 21  95  17  92 132] 2\n",
      "[ 17  92  12  89 132] 2\n",
      "[ 12  89   9  87 132] 2\n",
      "[  9  87   5  84 132] 2\n",
      "[  5  84   0  80 132] 2\n",
      "[  0  80  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1   0  71 132] 1\n",
      "[  0  71   6  67 132] 1\n",
      "[  6  67   9  65 132] 1\n",
      "[  9  65  15  61 132] 2\n",
      "[ 15  61  20  58 132] 1\n",
      "[ 20  58  24  55 132] 3\n",
      "[ 24  55  30  51 132] 3\n",
      "[ 30  51  33  49 132] 3\n",
      "[ 33  49  38  46 130] 3\n",
      "[ 38  46  44  42 124] 1\n",
      "[ 44  42  50  38 124] 1\n",
      "[ 50  38  56  34 111] 3\n",
      "[56 34 60 31 95] 3\n",
      "[60 31 65 28 77] 3\n",
      "[65 28 71 24 54] 3\n",
      "[71 24 77 20 31] 3\n",
      "[77 20 80 18 19] 3\n",
      "[80 18 86 14  0] 3\n",
      "[86 14 84 17  0] 3\n",
      "[84 17 81 21  0] 1\n",
      "[81 21 78 26  1] 2\n",
      "[78 26 76 29  9] 2\n",
      "[76 29 74 32 20] 2\n",
      "[74 32 72 35 30] 2\n",
      "[72 35 70 38 42] 2\n",
      "[70 38 66 44 66] 2\n",
      "[66 44 64 47 76] 2\n",
      "[64 47 61 51 94] 2\n",
      "[ 61  51  57  57 116] 2\n",
      "[ 57  57  53  63 132] 2\n",
      "[ 53  63  51  66 132] 2\n",
      "[ 51  66  49  69 132] 2\n",
      "[ 49  69  47  72 132] 2\n",
      "[ 47  72  43  78 132] 2\n",
      "[ 43  78  41  81 132] 2\n",
      "[ 41  81  37  87 132] 2\n",
      "[ 37  87  35  90 132] 2\n",
      "[ 35  90  32  95 132] 2\n",
      "[ 32  95  29  99 132] 2\n",
      "[ 29  99  27 102 132] 2\n",
      "[ 27 102  25 105 132] 2\n",
      "[ 25 105  22 110 132] 2\n",
      "[ 22 110  18 116 132] 2\n",
      "[ 18 116  15 120 132] 2\n",
      "[ 15 120  11 126 132] 2\n",
      "[ 11 126   7 132 132] 2\n",
      "[  7 132   3 138 132] 2\n",
      "[  3 138   1 141 132] 2\n",
      "[  1 141   0 138 132] 2\n",
      "[  0 138  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1   0 122 132] 1\n",
      "[  0 122   4 116 132] 1\n",
      "[  4 116   8 110 132] 1\n",
      "[  8 110  12 104 132] 2\n",
      "[ 12 104  16  98 132] 2\n",
      "[ 16  98  20  92 132] 2\n",
      "[ 20  92  23  87 132] 3\n",
      "[ 23  87  26  83 132] 2\n",
      "[ 26  83  30  77 132] 2\n",
      "[ 30  77  34  71 132] 3\n",
      "[ 34  71  37  66 132] 3\n",
      "[ 37  66  40  62 131] 3\n",
      "[ 40  62  44  56 117] 3\n",
      "[44 56 48 50 95] 3\n",
      "[48 50 52 44 71] 3\n",
      "[52 44 55 39 54] 3\n",
      "[55 39 57 36 43] 3\n",
      "[57 36 60 32 25] 3\n",
      "[60 32 62 29 13] 3\n",
      "[62 29 65 24  0] 3\n",
      "[65 24 67 21  0] 3\n",
      "[67 21 69 18  0] 3\n",
      "[69 18 71 15  0] 3\n",
      "[71 15 73 12  0] 3\n",
      "[73 12 77  6  0] 3\n",
      "[77  6 81  0  0] 3\n",
      "[81  0 85  3  0] 1\n",
      "[85  3 82  4  0] 1\n",
      "[82  4 76  2  0] 1\n",
      "[76  2 70  1  0] 1\n",
      "[70  1 66  0  0] 1\n",
      "[66  0 58  1  0] 1\n",
      "[58  1 50  3  0] 1\n",
      "[50  3 46  4  0] 1\n",
      "[46  4 40  5  4] 2\n",
      "[40  5 36  6 14] 2\n",
      "[36  6 32  7 25] 2\n",
      "[32  7 28  8 30] 3\n",
      "[28  8 24  9 24] 3\n",
      "[24  9 16 11  1] 3\n",
      "[16 11 10 13  1] 2\n",
      "[10 13  6 14  9] 2\n",
      "[ 6 14  0 15 11] 3\n",
      "[ 0 15 -1 -1  1] 1\n",
      "[-1 -1 -1 -1  9] 2\n",
      "[-1 -1  0 19 17] 1\n",
      "[ 0 19  4 20 19] 1\n",
      "[ 4 20 10 22 12] 3\n",
      "[10 22 18 24  2] 1\n",
      "[18 24 24 25  2] 1\n",
      "[24 25 28 26  2] 1\n",
      "[28 26 34 28  2] 1\n",
      "[34 28 42 30  2] 1\n",
      "[42 30 50 32  2] 1\n",
      "[50 32 54 33  2] 1\n",
      "[54 33 58 34  2] 1\n",
      "[58 34 62 35  2] 1\n",
      "[62 35 66 36  2] 1\n",
      "[66 36 74 38  2] 1\n",
      "[74 38 82 40  2] 1\n",
      "ok\n",
      "[82 40 -2 -2  5] 1\n",
      "ok\n",
      "[-2 -2 -2 -2  2] 1\n",
      "[-2 -2 -1 -1  2] 1\n",
      "[-1 -1 -1 -1  2] 1\n",
      "[-1 -1 -1 -1  2] 1\n",
      "[-1 -1 -1 -1  2] 1\n",
      "[-1 -1 -1 -1  2] 1\n",
      "[-1 -1 24 74  2] 1\n",
      "[24 74 26 76  2] 1\n",
      "[26 76 30 80 14] 2\n",
      "[30 80 34 84 25] 1\n",
      "[34 84 37 87 25] 1\n",
      "[37 87 40 90 27] 2\n",
      "[40 90 42 92 36] 2\n",
      "[42 92 46 96 60] 2\n",
      "[ 46  96  50 100  82] 2\n",
      "[ 50 100  53 103 100] 2\n",
      "[ 53 103  55 105 111] 2\n",
      "[ 55 105  57 107 122] 2\n",
      "[ 57 107  60 110 132] 2\n",
      "[ 60 110  63 113 132] 2\n",
      "[ 63 113  67 117 132] 2\n",
      "[ 67 117  71 121 132] 1\n",
      "[ 71 121  75 125 132] 1\n",
      "[ 75 125  77 127 132] 1\n",
      "[ 77 127  81 131 132] 2\n",
      "[ 81 131  84 134 132] 2\n",
      "[ 84 134  85 134 132] 2\n",
      "[ 85 134  83 131 132] 2\n",
      "[ 83 131  79 125 132] 2\n",
      "[ 79 125  77 122 132] 2\n",
      "[ 77 122  73 116 132] 2\n",
      "[ 73 116  71 113 132] 2\n",
      "[ 71 113  67 107 132] 2\n",
      "[ 67 107  64 103 132] 3\n",
      "[ 64 103  61  98 132] 3\n",
      "[ 61  98  59  95 132] 3\n",
      "[ 59  95  55  89 128] 3\n",
      "[ 55  89  51  83 105] 3\n",
      "[51 83 49 80 95] 3\n",
      "[49 80 47 77 89] 2\n",
      "[ 47  77  44  73 101] 2\n",
      "[ 44  73  40  67 109] 3\n",
      "[40 67 36 61 89] 3\n",
      "[36 61 34 58 77] 3\n",
      "[34 58 32 55 65] 3\n",
      "[32 55 28 49 43] 3\n",
      "[28 49 26 46 31] 3\n",
      "[26 46 23 41 13] 3\n",
      "[23 41 20 37  0] 3\n",
      "[20 37 16 31  0] 3\n",
      "[16 31 13 26  0] 3\n",
      "[13 26 10 22  0] 3\n",
      "[10 22  8 19  0] 3\n",
      "[ 8 19  6 16  0] 3\n",
      "[ 6 16  2 10  0] 3\n",
      "[ 2 10  0  7  0] 3\n",
      "[0 7 0 4 0] 3\n",
      "[ 0  4 -1 -1  0] 3\n",
      "[-1 -1 -1 -1  1] 2\n",
      "[-1 -1 -1 -1  7] 1\n",
      "[-1 -1 -1 -1  7] 1\n",
      "[-1 -1 -1 -1  7] 1\n",
      "[-1 -1  0 23  7] 1\n",
      "[ 0 23  1 26  7] 1\n",
      "[ 1 26  3 29  7] 3\n",
      "[ 3 29  5 32  0] 3\n",
      "[ 5 32  9 38  0] 3\n",
      "[ 9 38 13 44  0] 1\n",
      "[13 44 17 50  0] 1\n",
      "[17 50 20 55  0] 1\n",
      "[20 55 24 61  9] 2\n",
      "[24 61 28 67 19] 1\n",
      "[28 67 30 70 19] 1\n",
      "[30 70 33 74 26] 2\n",
      "[33 74 35 77 36] 2\n",
      "[35 77 37 80 48] 2\n",
      "[37 80 41 86 71] 2\n",
      "[41 86 44 91 80] 1\n",
      "[44 91 46 94 81] 2\n",
      "[ 46  94  50 100  84] 2\n",
      "[ 50 100  54 106  93] 1\n",
      "[ 54 106  58 112 106] 2\n",
      "[-1 -1 24 74 91] 1\n",
      "[24 74 27 77 91] 1\n",
      "[27 77 31 81 91] 1\n",
      "[31 81 34 84 91] 1\n",
      "[ 34  84  38  88 103] 2\n",
      "[ 38  88  40  90 111] 1\n",
      "[ 40  90  42  92 116] 2\n",
      "[ 42  92  46  96 132] 2\n",
      "[ 46  96  48  98 132] 1\n",
      "[ 48  98  51 101 132] 2\n",
      "[ 51 101  55 105 132] 2\n",
      "[ 55 105  57 107 132] 1\n",
      "[ 57 107  59 109 132] 2\n",
      "[ 59 109  63 113 132] 2\n",
      "[ 63 113  67 117 132] 1\n",
      "[ 67 117  70 120 132] 1\n",
      "[ 70 120  72 122 132] 2\n",
      "[ 72 122  74 124 132] 2\n",
      "[ 74 124  78 128 132] 2\n",
      "[ 78 128  82 132 132] 2\n",
      "[ 82 132  85 135 132] 2\n",
      "[ 85 135  83 131 132] 2\n",
      "[ 83 131  79 125 132] 2\n",
      "[ 79 125  77 122 132] 2\n",
      "[ 77 122  73 116 132] 2\n",
      "[ 73 116  71 113 132] 2\n",
      "[ 71 113  68 109 132] 2\n",
      "[ 68 109  66 106 132] 2\n",
      "[ 66 106  64 103 132] 2\n",
      "[ 64 103  60  97 132] 2\n",
      "[ 60  97  57  92 132] 3\n",
      "[ 57  92  54  88 132] 3\n",
      "[ 54  88  52  85 132] 3\n",
      "[ 52  85  48  79 128] 3\n",
      "[ 48  79  46  76 117] 3\n",
      "[46 76 42 70 95] 3\n",
      "[42 70 40 67 83] 3\n",
      "[40 67 38 64 71] 3\n",
      "[38 64 36 61 59] 3\n",
      "[36 61 33 56 43] 3\n",
      "[33 56 31 53 31] 3\n",
      "[31 53 28 49 13] 3\n",
      "[28 49 26 46  2] 3\n",
      "[26 46 22 40  0] 3\n",
      "[22 40 18 34  0] 3\n",
      "[18 34 16 31  0] 3\n",
      "[16 31 13 26  0] 3\n",
      "[13 26  9 20  0] 3\n",
      "[ 9 20  6 16  0] 3\n",
      "[ 6 16  3 11  0] 3\n",
      "[ 3 11  0  7  0] 3\n",
      "[ 0  7 -1 -1  0] 3\n",
      "[-1 -1 -1 -1  1] 2\n",
      "[-1 -1 -1 -1  7] 1\n",
      "[-1 -1  0 11  7] 1\n",
      "[ 0 11  3 17  7] 1\n",
      "[ 3 17  5 20  7] 1\n",
      "[ 5 20  7 23  4] 3\n",
      "[ 7 23 11 29  0] 3\n",
      "[11 29 14 34  0] 1\n",
      "[14 34 16 37  0] 1\n",
      "[16 37 20 43  9] 2\n",
      "[20 43 24 49 19] 1\n",
      "[24 49 28 55 19] 1\n",
      "[28 55 31 59 19] 1\n",
      "[31 59 33 62 19] 2\n",
      "[33 62 35 65 26] 2\n",
      "[35 65 39 71 48] 2\n",
      "[39 71 43 77 59] 1\n",
      "[43 77 46 82 59] 1\n",
      "[46 82 49 86 62] 2\n",
      "[49 86 51 89 71] 2\n",
      "[51 89 55 95 94] 2\n",
      "[ 55  95  59 101 110] 1\n",
      "[ 59 101  62 106 111] 1\n",
      "[ 62 106  66 112 117] 2\n",
      "[ 66 112  69 116 126] 1\n",
      "[ 69 116  72 121 131] 2\n",
      "[ 72 121  75 125 132] 2\n",
      "[ 75 125  78 130 132] 2\n",
      "[ 78 130  80 133 132] 2\n",
      "[ 80 133  84 139 132] 2\n",
      "[ 84 139  83 136 132] 2\n",
      "[ 83 136  77 132 132] 2\n",
      "[ 77 132  71 128 132] 2\n",
      "[ 71 128  66 125 132] 2\n",
      "[ 66 125  60 121 132] 2\n",
      "[ 60 121  56 118 132] 2\n",
      "[ 56 118  50 114 132] 2\n",
      "[ 50 114  45 111 132] 2\n",
      "[ 45 111  41 108 132] 2\n",
      "[ 41 108  38 106 132] 2\n",
      "[ 38 106  33 103 132] 2\n",
      "[ 33 103  27  99 132] 2\n",
      "[ 27  99  21  95 132] 2\n",
      "[ 21  95  18  93 132] 2\n",
      "[ 18  93  15  91 132] 2\n",
      "[ 15  91   9  87 132] 2\n",
      "[  9  87   3  83 132] 2\n",
      "[  3  83   0  81 132] 2\n",
      "[  0  81  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1   2  70 132] 1\n",
      "[  2  70   5  68 132] 1\n",
      "[  5  68   8  66 132] 2\n",
      "[  8  66  14  62 132] 2\n",
      "[ 14  62  18  59 132] 1\n",
      "[ 18  59  23  56 132] 3\n",
      "[ 23  56  29  52 132] 3\n",
      "[ 29  52  35  48 130] 3\n",
      "[ 35  48  38  46 126] 1\n",
      "[ 38  46  44  42 111] 3\n",
      "[44 42 48 39 95] 3\n",
      "[48 39 51 37 83] 3\n",
      "[51 37 54 35 71] 3\n",
      "[54 35 57 33 59] 3\n",
      "[57 33 63 29 37] 3\n",
      "[63 29 66 27 25] 3\n",
      "[66 27 72 23  2] 3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[72 23 75 21  0] 3\n",
      "[75 21 80 18  0] 3\n",
      "[80 18 86 14  0] 3\n",
      "[86 14 82 20  0] 3\n",
      "[82 20 78 26  9] 2\n",
      "[78 26 75 30 25] 2\n",
      "[75 30 72 35 42] 2\n",
      "[72 35 70 38 54] 2\n",
      "[70 38 67 42 71] 2\n",
      "[67 42 64 47 88] 2\n",
      "[ 64  47  62  50 100] 2\n",
      "[ 62  50  58  56 122] 2\n",
      "[ 58  56  55  60 132] 2\n",
      "[ 55  60  52  65 132] 2\n",
      "[ 52  65  48  71 132] 2\n",
      "[ 48  71  44  77 132] 2\n",
      "[ 44  77  40  83 132] 2\n",
      "[ 40  83  38  86 132] 2\n",
      "[ 38  86  36  89 132] 2\n",
      "[ 36  89  34  92 132] 2\n",
      "[ 34  92  32  95 132] 2\n",
      "[ 32  95  30  98 132] 2\n",
      "[ 30  98  26 104 132] 2\n",
      "[ 26 104  22 110 132] 2\n",
      "[ 22 110  19 114 132] 2\n",
      "[ 19 114  15 120 132] 2\n",
      "[ 15 120  13 123 132] 2\n",
      "[ 13 123  11 126 132] 2\n",
      "[ 11 126   8 131 132] 2\n",
      "[  8 131   5 135 132] 2\n",
      "[  5 135   1 141 132] 2\n",
      "[  1 141   0 138 132] 2\n",
      "[  0 138  -1  -1 132] 2\n",
      "[ -1  -1   0 126 132] 2\n",
      "[  0 126   1 120 132] 1\n",
      "[  1 120   5 114 132] 2\n",
      "[  5 114   7 111 132] 1\n",
      "[  7 111   9 108 132] 2\n",
      "[  9 108  11 105 132] 2\n",
      "[ 11 105  14 101 132] 2\n",
      "[ 14 101  18  95 132] 2\n",
      "[ 18  95  21  90 132] 2\n",
      "[ 21  90  25  84 132] 2\n",
      "[ 25  84  27  81 132] 3\n",
      "[ 27  81  29  78 132] 2\n",
      "[ 29  78  32  74 132] 2\n",
      "[ 32  74  36  68 132] 3\n",
      "[ 36  68  39  63 132] 3\n",
      "[ 39  63  42  59 132] 3\n",
      "[ 42  59  45  54 130] 3\n",
      "[ 45  54  47  51 123] 3\n",
      "[ 47  51  50  47 105] 3\n",
      "[50 47 52 44 95] 3\n",
      "[52 44 55 39 77] 3\n",
      "[55 39 58 35 59] 3\n",
      "[58 35 60 32 49] 3\n",
      "[60 32 64 26 25] 3\n",
      "[64 26 67 21  7] 3\n",
      "[67 21 69 18  0] 3\n",
      "[69 18 73 12  0] 3\n",
      "[73 12 77  6  0] 3\n",
      "[77  6 80  2  0] 3\n",
      "[80  2 83  0  0] 1\n",
      "[83  0 86  5  0] 1\n",
      "[86  5 78  3  0] 1\n",
      "[78  3 72  1  0] 1\n",
      "[72  1 68  0  0] 1\n",
      "[68  0 60  0  0] 1\n",
      "[60  0 56  1  0] 1\n",
      "[56  1 48  3  0] 1\n",
      "[48  3 42  5  0] 1\n",
      "[42  5 38  6  0] 2\n",
      "[38  6 30  8 14] 2\n",
      "[30  8 24  9 30] 2\n",
      "[24  9 20 10 42] 3\n",
      "[20 10 14 12 60] 3\n",
      "[14 12 10 13 57] 3\n",
      "[10 13  6 14 47] 3\n",
      "[ 6 14  0 16 25] 3\n",
      "[ 0 16 -1 -1 16] 1\n",
      "[-1 -1 -1 -1 21] 2\n",
      "[-1 -1  0 19 30] 1\n",
      "[ 0 19  8 21 35] 1\n",
      "[ 8 21 16 23 35] 1\n",
      "[16 23 20 24 35] 1\n",
      "[20 24 24 25 32] 3\n",
      "[24 25 32 27 13] 3\n",
      "[32 27 40 29  2] 1\n",
      "[40 29 48 31  2] 1\n",
      "[48 31 56 33  2] 1\n",
      "[56 33 60 34  2] 1\n",
      "[60 34 64 35  2] 1\n",
      "[64 35 68 36  2] 1\n",
      "[68 36 72 37  2] 1\n",
      "[72 37 78 39  2] 1\n",
      "[78 39 82 40  2] 1\n",
      "ok\n",
      "[82 40 -2 -2  5] 1\n",
      "[-2 -2 -1 -1  2] 1\n",
      "[-1 -1 -1 -1  2] 1\n",
      "[-1 -1 -1 -1  2] 1\n",
      "[-1 -1 -1 -1  2] 1\n",
      "[-1 -1 -1 -1  2] 1\n",
      "[-1 -1 25 75  2] 1\n",
      "[25 75 27 77  2] 1\n",
      "[27 77 31 81  5] 2\n",
      "[31 81 33 83 14] 1\n",
      "[33 83 37 87 30] 2\n",
      "[37 87 40 90 40] 1\n",
      "[40 90 42 92 44] 2\n",
      "[42 92 45 95 60] 2\n",
      "[45 95 48 98 76] 2\n",
      "[ 48  98  52 102 100] 2\n",
      "[ 52 102  56 106 122] 2\n",
      "[ 56 106  58 108 129] 1\n",
      "[ 58 108  60 110 132] 2\n",
      "[ 60 110  63 113 132] 2\n",
      "[ 63 113  67 117 132] 2\n",
      "[ 67 117  69 119 132] 1\n",
      "[ 69 119  73 123 132] 2\n",
      "[ 73 123  76 126 132] 1\n",
      "[ 76 126  80 130 132] 2\n",
      "[ 80 130  83 133 132] 2\n",
      "[ 83 133  85 134 132] 2\n",
      "[ 85 134  82 130 132] 2\n",
      "[ 82 130  79 125 132] 2\n",
      "[ 79 125  76 121 132] 2\n",
      "[ 76 121  72 115 132] 2\n",
      "[ 72 115  68 109 132] 2\n",
      "[ 68 109  64 103 132] 2\n",
      "[ 64 103  60  97 132] 3\n",
      "[ 60  97  57  92 132] 3\n",
      "[ 57  92  55  89 132] 3\n",
      "[ 55  89  53  86 130] 3\n",
      "[ 53  86  51  83 123] 3\n",
      "[ 51  83  49  80 111] 3\n",
      "[49 80 46 76 95] 3\n",
      "[46 76 44 73 83] 3\n",
      "[44 73 41 68 65] 3\n",
      "[41 68 39 65 54] 3\n",
      "[39 65 36 61 37] 3\n",
      "[36 61 32 55 13] 3\n",
      "[32 55 29 50  0] 3\n",
      "[29 50 25 44  0] 3\n",
      "[25 44 23 41  0] 3\n",
      "[23 41 21 38  0] 3\n",
      "[21 38 19 35  0] 3\n",
      "[19 35 15 29  0] 3\n",
      "[15 29 11 23  0] 3\n",
      "[11 23  7 17  0] 3\n",
      "[ 7 17  3 11  0] 3\n",
      "[ 3 11  0  5  0] 3\n",
      "[ 0  5 -1 -1  0] 3\n",
      "[-1 -1 -1 -1  0] 2\n",
      "[-1 -1 -1 -1  1] 1\n",
      "[-1 -1 -1 -1  1] 1\n",
      "[-1 -1 -1 -1  1] 1\n",
      "[-1 -1 -1 -1  1] 1\n",
      "[-1 -1  0 23  1] 1\n",
      "[ 0 23  2 28  1] 1\n",
      "[ 2 28  4 31  0] 3\n",
      "[ 4 31  7 35  0] 3\n",
      "[ 7 35 10 40  0] 3\n",
      "[10 40 13 44  0] 3\n",
      "[13 44 15 47  0] 3\n",
      "[15 47 18 52  0] 3\n",
      "[18 52 21 56  0] 1\n",
      "[21 56 23 59  1] 2\n",
      "[23 59 25 62  9] 2\n",
      "[25 62 28 67 25] 2\n",
      "[28 67 32 73 48] 2\n",
      "[32 73 34 76 60] 1\n",
      "[34 76 37 80 71] 2\n",
      "[37 80 39 83 82] 2\n",
      "[39 83 41 86 94] 2\n",
      "[ 41  86  43  89 106] 2\n",
      "[ 43  89  47  95 128] 2\n",
      "[ 47  95  49  98 132] 1\n",
      "[ 49  98  53 104 132] 2\n",
      "[ 53 104  57 110 132] 1\n",
      "[ 57 110  61 116 132] 1\n",
      "[ 61 116  64 121 132] 1\n",
      "[-1 -1 23 69 91] 1\n",
      "[23 69 26 66 91] 1\n",
      "[26 66 29 63 84] 3\n",
      "[29 63 31 61 73] 3\n",
      "[31 61 35 57 51] 3\n",
      "[35 57 39 53 27] 3\n",
      "[39 53 41 51 16] 3\n",
      "[41 51 45 47  0] 3\n",
      "[45 47 47 45  0] 3\n",
      "[47 45 49 43  0] 3\n",
      "[49 43 53 39  0] 3\n",
      "[53 39 56 36  0] 3\n",
      "[56 36 60 32  0] 3\n",
      "[60 32 62 30  0] 3\n",
      "[62 30 64 28  0] 3\n",
      "[64 28 67 25  0] 3\n",
      "[67 25 70 22  0] 3\n",
      "[70 22 74 18  0] 3\n",
      "[74 18 77 15  0] 3\n",
      "[77 15 80 12  0] 3\n",
      "[80 12 82 10  0] 3\n",
      "[82 10 85  7  0] 3\n",
      "[85  7 83  8  0] 1\n",
      "[83  8 80  6  0] 1\n",
      "[80  6 77  4  0] 3\n",
      "[77  4 72  1  0] 3\n",
      "[72  1 66  1  0] 3\n",
      "[66  1 63  3  0] 1\n",
      "[63  3 60  5  0] 1\n",
      "[60  5 57  7  0] 1\n",
      "[57  7 54  9  0] 1\n",
      "[54  9 48 13  4] 2\n",
      "[48 13 45 15 14] 2\n",
      "[45 15 41 18 30] 2\n",
      "[41 18 36 21 48] 2\n",
      "[36 21 33 23 60] 2\n",
      "[33 23 27 27 82] 2\n",
      "[ 27  27  21  31 106] 2\n",
      "[ 21  31  15  35 128] 2\n",
      "[ 15  35   9  39 132] 2\n",
      "[  9  39   3  43 132] 2\n",
      "[  3  43   0  46 132] 2\n",
      "[  0  46  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1   0  55 132] 1\n",
      "[  0  55   3  57 132] 1\n",
      "[  3  57   9  61 132] 2\n",
      "[  9  61  15  65 132] 1\n",
      "[ 15  65  21  69 132] 1\n",
      "[ 21  69  27  73 132] 1\n",
      "[ 27  73  30  75 132] 1\n",
      "[ 30  75  33  77 132] 2\n",
      "[ 33  77  39  81 132] 2\n",
      "[ 39  81  44  84 132] 1\n",
      "[ 44  84  47  86 132] 1\n",
      "[ 47  86  53  90 132] 1\n",
      "[ 53  90  56  92 132] 1\n",
      "[ 56  92  59  94 132] 1\n",
      "[ 59  94  65  98 132] 1\n",
      "[ 65  98  68 100 132] 1\n",
      "[ 68 100  74 104 132] 2\n",
      "[ 74 104  77 106 132] 1\n",
      "[ 77 106  80 108 132] 2\n",
      "[ 80 108  86 112 132] 2\n",
      "ok\n",
      "[ 86 112  -2  -2 129] 1\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  22   8 132] 1\n",
      "[ 22   8  24  10 132] 1\n",
      "[ 24  10  27  13 132] 1\n",
      "[ 27  13  29  15 132] 1\n",
      "[ 29  15  31  17 132] 1\n",
      "[ 31  17  33  19 132] 1\n",
      "[ 33  19  37  23 132] 1\n",
      "[ 37  23  41  27 132] 1\n",
      "[ 41  27  45  31 132] 1\n",
      "[ 45  31  47  33 132] 1\n",
      "[ 47  33  49  35 132] 1\n",
      "[ 49  35  51  37 132] 1\n",
      "[ 51  37  55  41 132] 1\n",
      "[ 55  41  59  45 132] 1\n",
      "[ 59  45  63  49 132] 1\n",
      "[ 63  49  65  51 132] 1\n",
      "[ 65  51  69  55 132] 1\n",
      "[ 69  55  73  59 132] 1\n",
      "[ 73  59  75  61 132] 1\n",
      "[ 75  61  77  63 132] 2\n",
      "[ 77  63  80  66 132] 2\n",
      "[ 80  66  84  70 132] 1\n",
      "[ 84  70  87  73 132] 1\n",
      "ok\n",
      "[ 87  73  -2  -2 119] 1\n",
      "ok\n",
      "[ -2  -2  -2  -2 132] 2\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  25  67 132] 1\n",
      "[ 25  67  27  65 132] 1\n",
      "[ 27  65  30  62 132] 3\n",
      "[ 30  62  32  60 132] 3\n",
      "[ 32  60  36  56 132] 3\n",
      "[ 36  56  39  53 128] 3\n",
      "[ 39  53  41  51 117] 3\n",
      "[ 41  51  44  48 100] 3\n",
      "[44 48 46 46 89] 3\n",
      "[46 46 50 42 65] 3\n",
      "[50 42 54 38 43] 3\n",
      "[54 38 57 35 25] 3\n",
      "[57 35 61 31  2] 3\n",
      "[61 31 63 29  0] 3\n",
      "[63 29 66 26  0] 3\n",
      "[66 26 70 22  0] 3\n",
      "[70 22 72 20  0] 3\n",
      "[72 20 74 18  0] 3\n",
      "[74 18 76 16  0] 3\n",
      "[76 16 78 14  0] 3\n",
      "[78 14 82 10  0] 3\n",
      "[82 10 84  8  0] 3\n",
      "[84  8 85  9  0] 1\n",
      "[85  9 82  7  0] 1\n",
      "[82  7 77  4  0] 3\n",
      "[77  4 73  1  0] 3\n",
      "[73  1 70  0  0] 3\n",
      "[70  0 64  3  0] 1\n",
      "[64  3 61  5  0] 1\n",
      "[61  5 58  7  0] 1\n",
      "[58  7 52 11  0] 1\n",
      "[52 11 49 13  1] 2\n",
      "[49 13 44 16 14] 2\n",
      "[44 16 40 19 30] 2\n",
      "[40 19 35 22 48] 2\n",
      "[35 22 29 26 71] 2\n",
      "[29 26 26 28 82] 2\n",
      "[26 28 23 30 94] 2\n",
      "[ 23  30  19  33 111] 2\n",
      "[ 19  33  16  35 122] 2\n",
      "[ 16  35  11  38 132] 2\n",
      "[ 11  38   7  41 132] 2\n",
      "[  7  41   4  43 132] 2\n",
      "[  4  43   0  47 132] 2\n",
      "[  0  47  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1   0  62 132] 1\n",
      "[  0  62   5  66 132] 1\n",
      "[  5  66  11  70 132] 1\n",
      "[ 11  70  14  72 132] 1\n",
      "[ 14  72  17  74 132] 2\n",
      "[ 17  74  22  77 132] 2\n",
      "[ 22  77  26  80 132] 1\n",
      "[ 26  80  32  84 132] 1\n",
      "[ 32  84  38  88 132] 1\n",
      "[ 38  88  41  90 132] 1\n",
      "[ 41  90  46  93 132] 2\n",
      "[ 46  93  50  96 132] 1\n",
      "[ 50  96  55  99 132] 1\n",
      "[ 55  99  58 101 132] 1\n",
      "[ 58 101  62 104 132] 1\n",
      "[ 62 104  67 107 132] 1\n",
      "[ 67 107  73 111 132] 1\n",
      "[ 73 111  79 115 132] 1\n",
      "[ 79 115  85 119 132] 1\n",
      "ok\n",
      "[ 85 119  -2  -2 128] 1\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  22   8 132] 1\n",
      "[ 22   8  25  11 132] 1\n",
      "[ 25  11  28  14 132] 1\n",
      "[ 28  14  31  17 132] 1\n",
      "[ 31  17  33  19 132] 1\n",
      "[ 33  19  37  23 132] 1\n",
      "[ 37  23  41  27 132] 1\n",
      "[ 41  27  45  31 132] 1\n",
      "[ 45  31  48  34 132] 1\n",
      "[ 48  34  51  37 132] 1\n",
      "[ 51  37  53  39 132] 1\n",
      "[ 53  39  57  43 132] 1\n",
      "[ 57  43  59  45 132] 1\n",
      "[ 59  45  63  49 132] 1\n",
      "[ 63  49  67  53 132] 1\n",
      "[ 67  53  71  57 132] 1\n",
      "[ 71  57  73  59 132] 1\n",
      "[ 73  59  77  63 132] 2\n",
      "[ 77  63  81  67 132] 1\n",
      "[ 81  67  85  71 132] 1\n",
      "[ 85  71  87  73 132] 1\n",
      "[ 87  73  -1  -1 119] 2\n",
      "ok\n",
      "[ -1  -1  -2  -2 125] 2\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  22  70 132] 1\n",
      "[ 22  70  26  66 132] 1\n",
      "[ 26  66  29  63 132] 3\n",
      "[ 29  63  32  60 132] 3\n",
      "[ 32  60  36  56 131] 3\n",
      "[ 36  56  38  54 128] 3\n",
      "[ 38  54  42  50 105] 3\n",
      "[42 50 44 48 95] 3\n",
      "[44 48 47 45 77] 3\n",
      "[47 45 49 43 65] 3\n",
      "[49 43 53 39 43] 3\n",
      "[53 39 57 35 19] 3\n",
      "[57 35 60 32  2] 3\n",
      "[60 32 62 30  0] 3\n",
      "[62 30 65 27  0] 3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[65 27 67 25  0] 3\n",
      "[67 25 69 23  0] 3\n",
      "[69 23 71 21  0] 3\n",
      "[71 21 75 17  0] 3\n",
      "[75 17 78 14  0] 3\n",
      "[78 14 81 11  0] 3\n",
      "[81 11 85  7  0] 3\n",
      "[85  7 84  9  0] 1\n",
      "[84  9 78  5  0] 1\n",
      "[78  5 74  2  0] 3\n",
      "[74  2 69  0  0] 3\n",
      "[69  0 66  1  0] 1\n",
      "[66  1 60  5  0] 1\n",
      "[60  5 57  7  0] 1\n",
      "[57  7 51 11  0] 1\n",
      "[51 11 48 13  1] 2\n",
      "[48 13 42 17 20] 2\n",
      "[42 17 36 21 42] 2\n",
      "[36 21 33 23 54] 2\n",
      "[33 23 30 25 66] 2\n",
      "[30 25 24 29 88] 2\n",
      "[ 24  29  20  32 106] 2\n",
      "[ 20  32  14  36 128] 2\n",
      "[ 14  36   8  40 132] 2\n",
      "[  8  40   5  42 132] 2\n",
      "[  5  42   2  44 132] 2\n",
      "[  2  44   0  47 132] 2\n",
      "[  0  47  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1   0  66 132] 1\n",
      "[  0  66   3  69 132] 1\n",
      "[  3  69   6  71 132] 2\n",
      "[  6  71  11  74 132] 2\n",
      "[ 11  74  15  77 132] 1\n",
      "[ 15  77  21  81 132] 1\n",
      "[ 21  81  27  85 132] 1\n",
      "[ 27  85  33  89 132] 1\n",
      "[ 33  89  38  92 132] 1\n",
      "[ 38  92  41  94 132] 1\n",
      "[ 41  94  45  97 132] 2\n",
      "[ 45  97  48  99 132] 1\n",
      "[ 48  99  51 101 132] 2\n",
      "[ 51 101  57 105 132] 2\n",
      "[ 57 105  62 108 132] 1\n",
      "[ 62 108  65 110 132] 1\n",
      "[ 65 110  69 113 132] 2\n",
      "[ 69 113  75 117 132] 1\n",
      "[ 75 117  81 121 132] 1\n",
      "[ 81 121  84 123 132] 1\n",
      "[ 84 123  87 125 132] 2\n",
      "ok\n",
      "[ 87 125  -2  -2 130] 2\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[-1 -1 23 69 91] 1\n",
      "[23 69 26 66 91] 1\n",
      "[26 66 29 63 91] 3\n",
      "[29 63 33 59 84] 3\n",
      "[33 59 35 57 73] 3\n",
      "[35 57 38 54 57] 3\n",
      "[38 54 42 50 33] 3\n",
      "[42 50 44 48 21] 3\n",
      "[44 48 47 45  5] 3\n",
      "[47 45 49 43  0] 3\n",
      "[49 43 52 40  0] 3\n",
      "[52 40 56 36  0] 3\n",
      "[56 36 60 32  0] 3\n",
      "[60 32 64 28  0] 3\n",
      "[64 28 66 26  0] 3\n",
      "[66 26 70 22  0] 3\n",
      "[70 22 72 20  0] 3\n",
      "[72 20 74 18  0] 3\n",
      "[74 18 78 14  0] 3\n",
      "[78 14 80 12  0] 3\n",
      "[80 12 83  9  0] 3\n",
      "[83  9 85  7  0] 3\n",
      "[85  7 81  7  0] 1\n",
      "[81  7 77  4  0] 1\n",
      "[77  4 71  0  0] 3\n",
      "[71  0 68  0  0] 3\n",
      "[68  0 65  2  0] 1\n",
      "[65  2 62  4  0] 1\n",
      "[62  4 59  6  0] 1\n",
      "[59  6 53 10  0] 1\n",
      "[53 10 48 13  1] 2\n",
      "[48 13 44 16 14] 2\n",
      "[44 16 41 18 25] 2\n",
      "[41 18 36 21 42] 2\n",
      "[36 21 33 23 48] 3\n",
      "[33 23 27 27 30] 3\n",
      "[27 27 21 31 31] 2\n",
      "[21 31 15 35 54] 2\n",
      "[15 35 12 37 66] 2\n",
      "[12 37  9 39 76] 2\n",
      "[ 9 39  5 42 94] 2\n",
      "[  5  42   0  46 116] 2\n",
      "[  0  46  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1   0  55 132] 1\n",
      "[  0  55   6  59 132] 1\n",
      "[  6  59  12  63 132] 1\n",
      "[ 12  63  15  65 132] 1\n",
      "[ 15  65  20  68 132] 2\n",
      "[ 20  68  24  71 132] 1\n",
      "[ 24  71  30  75 132] 1\n",
      "[ 30  75  35  78 132] 1\n",
      "[ 35  78  41  82 132] 1\n",
      "[ 41  82  44  84 132] 1\n",
      "[ 44  84  50  88 132] 1\n",
      "[ 50  88  54  91 132] 1\n",
      "[ 54  91  59  94 132] 1\n",
      "[ 59  94  65  98 132] 1\n",
      "[ 65  98  68 100 132] 1\n",
      "[ 68 100  74 104 132] 2\n",
      "[ 74 104  80 108 132] 1\n",
      "[ 80 108  86 112 132] 1\n",
      "[ 86 112  -1  -1 127] 1\n",
      "ok\n",
      "[ -1  -1  -2  -2 129] 1\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  24  10 132] 1\n",
      "[ 24  10  26  12 132] 1\n",
      "[ 26  12  28  14 132] 1\n",
      "[ 28  14  32  18 132] 1\n",
      "[ 32  18  35  21 132] 1\n",
      "[ 35  21  39  25 132] 1\n",
      "[ 39  25  42  28 132] 1\n",
      "[ 42  28  44  30 132] 1\n",
      "[ 44  30  47  33 132] 1\n",
      "[ 47  33  49  35 132] 1\n",
      "[ 49  35  53  39 132] 1\n",
      "[ 53  39  55  41 132] 1\n",
      "[ 55  41  58  44 132] 1\n",
      "[ 58  44  62  48 132] 1\n",
      "[ 62  48  65  51 132] 1\n",
      "[ 65  51  67  53 132] 1\n",
      "[ 67  53  71  57 132] 1\n",
      "[ 71  57  74  60 132] 1\n",
      "[ 74  60  77  63 132] 1\n",
      "[ 77  63  80  66 132] 1\n",
      "[ 80  66  82  68 132] 1\n",
      "[ 82  68  84  70 132] 2\n",
      "[ 84  70  88  74 124] 2\n",
      "ok\n",
      "[ 88  74  -2  -2 125] 1\n",
      "ok\n",
      "[ -2  -2  -2  -2 132] 2\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  23  69 132] 1\n",
      "[ 23  69  27  65 132] 1\n",
      "[ 27  65  30  62 132] 3\n",
      "[ 30  62  33  59 132] 3\n",
      "[ 33  59  36  56 132] 3\n",
      "[ 36  56  38  54 130] 3\n",
      "[ 38  54  41  51 117] 3\n",
      "[41 51 45 47 95] 3\n",
      "[45 47 48 44 77] 3\n",
      "[48 44 51 41 59] 3\n",
      "[51 41 53 39 49] 3\n",
      "[53 39 57 35 25] 3\n",
      "[57 35 60 32  7] 3\n",
      "[60 32 63 29  0] 3\n",
      "[63 29 65 27  0] 3\n",
      "[65 27 69 23  0] 3\n",
      "[69 23 71 21  0] 3\n",
      "[71 21 75 17  0] 3\n",
      "[75 17 78 14  0] 3\n",
      "[78 14 81 11  0] 3\n",
      "[81 11 85  7  0] 3\n",
      "[85  7 85  9  0] 1\n",
      "[85  9 82  7  0] 1\n",
      "[82  7 77  4  0] 3\n",
      "[77  4 73  1  0] 3\n",
      "[73  1 70  0  0] 3\n",
      "[70  0 64  3  0] 1\n",
      "[64  3 61  5  0] 1\n",
      "[61  5 58  7  0] 1\n",
      "[58  7 53 10  0] 1\n",
      "[53 10 50 12  1] 2\n",
      "[50 12 47 14  9] 2\n",
      "[47 14 44 16 20] 2\n",
      "[44 16 40 19 36] 2\n",
      "[40 19 37 21 48] 2\n",
      "[37 21 31 25 46] 3\n",
      "[31 25 25 29 38] 2\n",
      "[25 29 20 32 54] 2\n",
      "[20 32 16 35 71] 2\n",
      "[16 35 10 39 94] 2\n",
      "[ 10  39   7  41 106] 2\n",
      "[  7  41   2  44 122] 2\n",
      "[  2  44  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1   0  62 132] 1\n",
      "[  0  62   5  66 132] 1\n",
      "[  5  66   8  68 132] 1\n",
      "[  8  68  11  70 132] 2\n",
      "[ 11  70  17  74 132] 2\n",
      "[ 17  74  20  76 132] 1\n",
      "[ 20  76  23  78 132] 2\n",
      "[ 23  78  29  82 132] 2\n",
      "[ 29  82  34  85 132] 1\n",
      "[ 34  85  38  88 132] 1\n",
      "[ 38  88  43  91 132] 1\n",
      "[ 43  91  47  94 132] 1\n",
      "[ 47  94  52  97 132] 1\n",
      "[ 52  97  56 100 132] 1\n",
      "[ 56 100  61 103 132] 1\n",
      "[ 61 103  64 105 132] 1\n",
      "[ 64 105  68 108 132] 1\n",
      "[ 68 108  71 110 132] 1\n",
      "[ 71 110  74 112 132] 2\n",
      "[ 74 112  77 114 132] 2\n",
      "[ 77 114  83 118 132] 2\n",
      "[ 83 118  88 121 130] 1\n",
      "ok\n",
      "[ 88 121  -2  -2 130] 1\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  22   8 132] 1\n",
      "[ 22   8  26  12 132] 1\n",
      "[ 26  12  29  15 132] 1\n",
      "[ 29  15  31  17 132] 1\n",
      "[ 31  17  35  21 132] 1\n",
      "[ 35  21  38  24 132] 1\n",
      "[ 38  24  40  26 132] 1\n",
      "[ 40  26  44  30 132] 1\n",
      "[ 44  30  47  33 132] 1\n",
      "[ 47  33  51  37 132] 1\n",
      "[ 51  37  54  40 132] 1\n",
      "[ 54  40  58  44 132] 1\n",
      "[ 58  44  61  47 132] 1\n",
      "[ 61  47  64  50 132] 1\n",
      "[ 64  50  67  53 132] 1\n",
      "[ 67  53  69  55 132] 1\n",
      "[ 69  55  71  57 132] 1\n",
      "[ 71  57  73  59 132] 2\n",
      "[ 73  59  75  61 132] 2\n",
      "[ 75  61  79  65 132] 2\n",
      "[ 79  65  81  67 132] 1\n",
      "[ 81  67  85  71 132] 2\n",
      "[ 85  71  87  73 132] 1\n",
      "ok\n",
      "[ 87  73  -2  -2 119] 2\n",
      "ok\n",
      "[ -2  -2  -2  -2 132] 2\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  22  70 132] 1\n",
      "[ 22  70  24  68 132] 1\n",
      "[ 24  68  27  65 132] 2\n",
      "[ 27  65  31  61 132] 3\n",
      "[ 31  61  34  58 132] 3\n",
      "[ 34  58  38  54 130] 3\n",
      "[ 38  54  40  52 123] 3\n",
      "[ 40  52  44  48 100] 3\n",
      "[44 48 48 44 77] 3\n",
      "[48 44 50 42 65] 3\n",
      "[50 42 54 38 43] 3\n",
      "[54 38 56 36 31] 3\n",
      "[56 36 59 33 13] 3\n",
      "[59 33 62 30  0] 3\n",
      "[62 30 64 28  0] 3\n",
      "[64 28 67 25  0] 3\n",
      "[67 25 69 23  0] 3\n",
      "[69 23 73 19  0] 3\n",
      "[73 19 77 15  0] 3\n",
      "[77 15 79 13  0] 3\n",
      "[79 13 82 10  0] 3\n",
      "[82 10 85  7  0] 3\n",
      "[85  7 81  7  0] 1\n",
      "[81  7 78  5  0] 1\n",
      "[78  5 72  1  0] 3\n",
      "[72  1 68  0  0] 3\n",
      "[68  0 63  3  0] 1\n",
      "[63  3 60  5  0] 1\n",
      "[60  5 56  8  0] 1\n",
      "[56  8 50 12  0] 1\n",
      "[50 12 47 14  1] 2\n",
      "[47 14 42 17 14] 2\n",
      "[42 17 39 19 25] 2\n",
      "[39 19 35 22 42] 2\n",
      "[35 22 30 25 45] 3\n",
      "[30 25 26 28 44] 2\n",
      "[26 28 21 31 60] 2\n",
      "[21 31 17 34 76] 2\n",
      "[ 17  34  11  38 100] 2\n",
      "[ 11  38   8  40 111] 2\n",
      "[  8  40   2  44 131] 2\n",
      "[  2  44  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1   0  65 132] 1\n",
      "[  0  65   2  68 132] 1\n",
      "[  2  68   8  72 132] 2\n",
      "[  8  72  14  76 132] 1\n",
      "[ 14  76  20  80 132] 1\n",
      "[ 20  80  24  83 132] 1\n",
      "[ 24  83  27  85 132] 1\n",
      "[ 27  85  32  88 132] 2\n",
      "[ 32  88  35  90 132] 1\n",
      "[ 35  90  41  94 132] 2\n",
      "[ 41  94  45  97 132] 1\n",
      "[ 45  97  51 101 132] 1\n",
      "[ 51 101  54 103 132] 1\n",
      "[ 54 103  59 106 132] 2\n",
      "[ 59 106  65 110 132] 1\n",
      "[ 65 110  69 113 132] 1\n",
      "[ 69 113  75 117 132] 1\n",
      "[ 75 117  81 121 132] 1\n",
      "[ 81 121  84 123 132] 1\n",
      "[ 84 123  -1  -1 129] 2\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[-1 -1 23 69 91] 1\n",
      "[23 69 26 66 91] 1\n",
      "[26 66 30 62 79] 3\n",
      "[30 62 33 59 63] 3\n",
      "[33 59 35 57 51] 3\n",
      "[35 57 37 55 39] 3\n",
      "[37 55 39 53 27] 3\n",
      "[39 53 41 51 16] 3\n",
      "[41 51 44 48  0] 3\n",
      "[44 48 46 46  0] 3\n",
      "[46 46 49 43  0] 3\n",
      "[49 43 53 39  0] 3\n",
      "[53 39 56 36  0] 3\n",
      "[56 36 60 32  0] 3\n",
      "[60 32 62 30  0] 3\n",
      "[62 30 64 28  0] 3\n",
      "[64 28 68 24  0] 3\n",
      "[68 24 71 21  0] 3\n",
      "[71 21 74 18  0] 3\n",
      "[74 18 78 14  0] 3\n",
      "[78 14 80 12  0] 3\n",
      "[80 12 82 10  0] 3\n",
      "[82 10 85  7  0] 3\n",
      "[85  7 81  7  0] 1\n",
      "[81  7 75  3  0] 1\n",
      "[75  3 69  0  0] 3\n",
      "[69  0 66  1  0] 1\n",
      "[66  1 62  4  0] 1\n",
      "[62  4 56  8  0] 1\n",
      "[56  8 50 12  0] 1\n",
      "[50 12 47 14  1] 2\n",
      "[47 14 44 16  9] 2\n",
      "[44 16 38 20 30] 2\n",
      "[38 20 33 23 48] 2\n",
      "[33 23 29 26 66] 2\n",
      "[29 26 26 28 76] 2\n",
      "[ 26  28  20  32 100] 2\n",
      "[ 20  32  15  35 116] 2\n",
      "[ 15  35  11  38 131] 2\n",
      "[ 11  38   6  41 132] 2\n",
      "[  6  41   0  45 132] 2\n",
      "[  0  45   0  47 132] 2\n",
      "[  0  47  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1   0  54 132] 1\n",
      "[  0  54   2  56 132] 1\n",
      "[  2  56   5  58 132] 2\n",
      "[  5  58   8  60 132] 2\n",
      "[  8  60  14  64 132] 2\n",
      "[ 14  64  17  66 132] 1\n",
      "[ 17  66  23  70 132] 2\n",
      "[ 23  70  29  74 132] 1\n",
      "[ 29  74  35  78 132] 1\n",
      "[ 35  78  41  82 132] 1\n",
      "[ 41  82  45  85 132] 1\n",
      "[ 45  85  51  89 132] 1\n",
      "[ 51  89  57  93 132] 1\n",
      "[ 57  93  62  96 132] 1\n",
      "[ 62  96  65  98 132] 1\n",
      "[ 65  98  69 101 132] 2\n",
      "[ 69 101  75 105 132] 1\n",
      "[ 75 105  80 108 132] 1\n",
      "[ 80 108  83 110 132] 1\n",
      "[ 83 110  87 113 132] 2\n",
      "ok\n",
      "[ 87 113  -2  -2 129] 1\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  22   8 132] 1\n",
      "[ 22   8  25  11 132] 1\n",
      "[ 25  11  28  14 132] 1\n",
      "[ 28  14  32  18 132] 1\n",
      "[ 32  18  35  21 132] 1\n",
      "[ 35  21  39  25 132] 1\n",
      "[ 39  25  41  27 132] 1\n",
      "[ 41  27  45  31 132] 1\n",
      "[ 45  31  47  33 132] 1\n",
      "[ 47  33  49  35 132] 1\n",
      "[ 49  35  51  37 132] 1\n",
      "[ 51  37  55  41 132] 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 55  41  59  45 132] 1\n",
      "[ 59  45  63  49 132] 1\n",
      "[ 63  49  67  53 132] 1\n",
      "[ 67  53  69  55 132] 1\n",
      "[ 69  55  73  59 132] 1\n",
      "[ 73  59  75  61 132] 1\n",
      "[ 75  61  78  64 132] 2\n",
      "[ 78  64  81  67 132] 1\n",
      "[ 81  67  85  71 132] 1\n",
      "[ 85  71  -1  -1 119] 1\n",
      "ok\n",
      "[ -1  -1  -2  -2 125] 2\n",
      "ok\n",
      "[ -2  -2  -2  -2 132] 1\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  23  69 132] 1\n",
      "[ 23  69  27  65 132] 1\n",
      "[ 27  65  30  62 132] 3\n",
      "[ 30  62  34  58 132] 3\n",
      "[ 34  58  36  56 132] 3\n",
      "[ 36  56  38  54 130] 3\n",
      "[ 38  54  40  52 123] 3\n",
      "[ 40  52  43  49 105] 3\n",
      "[43 49 47 45 83] 3\n",
      "[47 45 51 41 59] 3\n",
      "[51 41 55 37 37] 3\n",
      "[55 37 57 35 25] 3\n",
      "[57 35 59 33 13] 3\n",
      "[59 33 63 29  0] 3\n",
      "[63 29 65 27  0] 3\n",
      "[65 27 68 24  0] 3\n",
      "[68 24 71 21  0] 3\n",
      "[71 21 73 19  0] 3\n",
      "[73 19 75 17  0] 3\n",
      "[75 17 78 14  0] 3\n",
      "[78 14 81 11  0] 3\n",
      "[81 11 84  8  0] 3\n",
      "[84  8 83  8  0] 1\n",
      "[83  8 80  6  0] 1\n",
      "[80  6 74  2  0] 3\n",
      "[74  2 70  0  0] 3\n",
      "[70  0 65  2  0] 1\n",
      "[65  2 62  4  0] 1\n",
      "[62  4 58  7  0] 1\n",
      "[58  7 52 11  0] 1\n",
      "[52 11 47 14  4] 2\n",
      "[47 14 43 17 20] 2\n",
      "[43 17 40 19 30] 2\n",
      "[40 19 35 22 48] 2\n",
      "[35 22 31 25 66] 2\n",
      "[31 25 26 28 82] 2\n",
      "[ 26  28  22  31 100] 2\n",
      "[ 22  31  17  34 116] 2\n",
      "[ 17  34  11  38 132] 2\n",
      "[ 11  38   7  41 132] 2\n",
      "[  7  41   1  45 132] 2\n",
      "[  1  45  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1   1  63 132] 1\n",
      "[  1  63   7  67 132] 1\n",
      "[  7  67  11  70 132] 1\n",
      "[ 11  70  16  73 132] 1\n",
      "[ 16  73  22  77 132] 1\n",
      "[ 22  77  25  79 132] 1\n",
      "[ 25  79  29  82 132] 2\n",
      "[ 29  82  32  84 132] 1\n",
      "[ 32  84  35  86 132] 2\n",
      "[ 35  86  38  88 132] 2\n",
      "[ 38  88  44  92 132] 2\n",
      "[ 44  92  50  96 132] 1\n",
      "[ 50  96  56 100 132] 1\n",
      "[ 56 100  61 103 132] 1\n",
      "[ 61 103  65 106 132] 1\n",
      "[ 65 106  70 109 132] 1\n",
      "[ 70 109  73 111 132] 1\n",
      "[ 73 111  76 113 132] 2\n",
      "[ 76 113  80 116 132] 2\n",
      "[ 80 116  83 118 132] 2\n",
      "[ 83 118  -1  -1 128] 2\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  24  10 132] 1\n",
      "[ 24  10  26  12 132] 1\n",
      "[ 26  12  30  16 132] 1\n",
      "[ 30  16  33  19 132] 1\n",
      "[ 33  19  36  22 132] 1\n",
      "[ 36  22  39  25 132] 1\n",
      "[ 39  25  43  29 132] 1\n",
      "[ 43  29  45  31 132] 1\n",
      "[ 45  31  48  34 132] 1\n",
      "[ 48  34  52  38 132] 1\n",
      "[ 52  38  55  41 132] 1\n",
      "[ 55  41  57  43 132] 1\n",
      "[ 57  43  61  47 132] 1\n",
      "[ 61  47  65  51 132] 1\n",
      "[ 65  51  68  54 132] 1\n",
      "[ 68  54  71  57 132] 1\n",
      "[ 71  57  75  61 132] 1\n",
      "[ 75  61  77  63 132] 1\n",
      "[ 77  63  81  67 132] 2\n",
      "[ 81  67  83  69 132] 1\n",
      "[ 83  69  86  72 132] 2\n",
      "ok\n",
      "[ 86  72  -2  -2 119] 1\n",
      "ok\n",
      "[ -2  -2  -2  -2 132] 2\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  23  69 132] 1\n",
      "[ 23  69  27  65 132] 1\n",
      "[ 27  65  31  61 132] 3\n",
      "[ 31  61  33  59 132] 3\n",
      "[ 33  59  37  55 132] 3\n",
      "[ 37  55  39  53 130] 3\n",
      "[ 39  53  42  50 117] 3\n",
      "[ 42  50  45  47 100] 3\n",
      "[45 47 47 45 89] 3\n",
      "[47 45 51 41 65] 3\n",
      "[51 41 54 38 49] 3\n",
      "[54 38 58 34 25] 3\n",
      "[58 34 62 30  2] 3\n",
      "[62 30 65 27  0] 3\n",
      "[65 27 67 25  0] 3\n",
      "[67 25 69 23  0] 3\n",
      "[69 23 72 20  0] 3\n",
      "[72 20 76 16  0] 3\n",
      "[76 16 80 12  0] 3\n",
      "[80 12 83  9  0] 3\n",
      "[83  9 84  9  0] 3\n",
      "[84  9 80  6  0] 1\n",
      "[80  6 75  3  0] 3\n",
      "[75  3 69  0  0] 3\n",
      "[69  0 63  3  0] 1\n",
      "[63  3 57  7  0] 1\n",
      "[57  7 53 10  0] 1\n",
      "[53 10 48 13  4] 2\n",
      "[48 13 45 15 14] 2\n",
      "[45 15 39 19 36] 2\n",
      "[39 19 35 22 54] 2\n",
      "[35 22 32 24 66] 2\n",
      "[32 24 27 27 82] 2\n",
      "[ 27  27  21  31 106] 2\n",
      "[ 21  31  18  33 116] 2\n",
      "[ 18  33  14  36 131] 3\n",
      "[ 14  36  11  38 132] 2\n",
      "[ 11  38   8  40 132] 2\n",
      "[  8  40   2  44 132] 2\n",
      "[  2  44   0  46 132] 2\n",
      "[  0  46  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1   0  65 132] 1\n",
      "[  0  65   3  69 132] 1\n",
      "[  3  69   6  71 132] 2\n",
      "[  6  71   9  73 132] 2\n",
      "[  9  73  15  77 132] 2\n",
      "[ 15  77  20  80 132] 1\n",
      "[ 20  80  26  84 132] 1\n",
      "[ 26  84  30  87 132] 1\n",
      "[ 30  87  36  91 132] 1\n",
      "[ 36  91  39  93 132] 1\n",
      "[ 39  93  45  97 132] 2\n",
      "[ 45  97  48  99 132] 1\n",
      "[ 48  99  53 102 132] 2\n",
      "[ 53 102  56 104 132] 1\n",
      "[ 56 104  60 107 132] 2\n",
      "[ 60 107  65 110 132] 1\n",
      "[ 65 110  68 112 132] 1\n",
      "[ 68 112  72 115 132] 2\n",
      "[ 72 115  78 119 132] 1\n",
      "[ 78 119  81 121 132] 1\n",
      "[ 81 121  84 123 132] 2\n",
      "[ 84 123  87 125 132] 2\n",
      "ok\n",
      "[ 87 125  -2  -2 130] 2\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[-1 -1 23 69 91] 1\n",
      "[23 69 26 66 91] 1\n",
      "[26 66 29 63 84] 3\n",
      "[29 63 33 59 63] 3\n",
      "[33 59 35 57 51] 3\n",
      "[35 57 37 55 39] 3\n",
      "[37 55 39 53 27] 3\n",
      "[39 53 43 49  5] 3\n",
      "[43 49 46 46  0] 3\n",
      "[46 46 48 44  0] 3\n",
      "[48 44 50 42  0] 3\n",
      "[50 42 53 39  0] 3\n",
      "[53 39 55 37  0] 3\n",
      "[55 37 57 35  0] 3\n",
      "[57 35 59 33  0] 3\n",
      "[59 33 61 31  0] 3\n",
      "[61 31 64 28  0] 3\n",
      "[64 28 68 24  0] 3\n",
      "[68 24 72 20  0] 3\n",
      "[72 20 74 18  0] 3\n",
      "[74 18 78 14  0] 3\n",
      "[78 14 80 12  0] 3\n",
      "[80 12 82 10  0] 3\n",
      "[82 10 84  8  0] 3\n",
      "[84  8 84  9  0] 1\n",
      "[84  9 78  5  0] 1\n",
      "[78  5 74  2  0] 3\n",
      "[74  2 69  0  0] 3\n",
      "[69  0 63  3  0] 1\n",
      "[63  3 57  7  0] 1\n",
      "[57  7 54  9  0] 1\n",
      "[54  9 51 11  1] 2\n",
      "[51 11 48 13  9] 2\n",
      "[48 13 44 16 25] 2\n",
      "[44 16 39 19 42] 2\n",
      "[39 19 35 22 60] 2\n",
      "[35 22 32 24 71] 2\n",
      "[32 24 27 27 88] 2\n",
      "[ 27  27  23  30 106] 2\n",
      "[ 23  30  20  32 116] 2\n",
      "[ 20  32  14  36 115] 3\n",
      "[ 14  36   8  40 108] 2\n",
      "[  8  40   3  43 122] 2\n",
      "[  3  43   0  47 132] 2\n",
      "[  0  47  -1  -1 132] 2\n",
      "[ -1  -1   0  55 132] 3\n",
      "[  0  55   6  59 132] 1\n",
      "[  6  59  12  63 132] 1\n",
      "[ 12  63  18  67 132] 1\n",
      "[ 18  67  21  69 132] 1\n",
      "[ 21  69  26  72 132] 2\n",
      "[ 26  72  29  74 132] 1\n",
      "[ 29  74  35  78 132] 2\n",
      "[ 35  78  38  80 132] 1\n",
      "[ 38  80  41  82 132] 1\n",
      "[ 41  82  44  84 132] 1\n",
      "[ 44  84  50  88 132] 1\n",
      "[ 50  88  56  92 132] 1\n",
      "[ 56  92  59  94 132] 1\n",
      "[ 59  94  65  98 132] 1\n",
      "[ 65  98  71 102 132] 1\n",
      "[ 71 102  75 105 132] 1\n",
      "[ 75 105  80 108 132] 1\n",
      "[ 80 108  84 111 132] 1\n",
      "[ 84 111  87 113 132] 1\n",
      "ok\n",
      "[ 87 113  -2  -2 129] 2\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  23   9 132] 1\n",
      "[ 23   9  25  11 132] 1\n",
      "[ 25  11  28  14 132] 1\n",
      "[ 28  14  30  16 132] 1\n",
      "[ 30  16  33  19 132] 1\n",
      "[ 33  19  36  22 132] 1\n",
      "[ 36  22  40  26 132] 1\n",
      "[ 40  26  42  28 132] 1\n",
      "[ 42  28  45  31 132] 1\n",
      "[ 45  31  48  34 132] 1\n",
      "[ 48  34  51  37 132] 1\n",
      "[ 51  37  53  39 132] 1\n",
      "[ 53  39  57  43 132] 1\n",
      "[ 57  43  60  46 132] 1\n",
      "[ 60  46  63  49 132] 1\n",
      "[ 63  49  67  53 132] 1\n",
      "[ 67  53  70  56 132] 1\n",
      "[ 70  56  73  59 132] 1\n",
      "[ 73  59  76  62 132] 1\n",
      "[ 76  62  80  66 132] 1\n",
      "[ 80  66  83  69 132] 1\n",
      "[ 83  69  86  72 132] 1\n",
      "[ 86  72  88  74 124] 1\n",
      "ok\n",
      "[ 88  74  -2  -2 119] 2\n",
      "ok\n",
      "[ -2  -2  -2  -2 125] 2\n",
      "ok\n",
      "[ -2  -2  -2  -2 132] 1\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  22  70 132] 1\n",
      "[ 22  70  26  66 132] 1\n",
      "[ 26  66  29  63 132] 3\n",
      "[ 29  63  33  59 132] 3\n",
      "[ 33  59  36  56 132] 3\n",
      "[ 36  56  38  54 132] 3\n",
      "[ 38  54  41  51 128] 3\n",
      "[ 41  51  44  48 111] 3\n",
      "[ 44  48  46  46 100] 3\n",
      "[46 46 50 42 77] 3\n",
      "[50 42 53 39 59] 3\n",
      "[53 39 57 35 37] 3\n",
      "[57 35 61 31 13] 3\n",
      "[61 31 65 27  0] 3\n",
      "[65 27 69 23  0] 3\n",
      "[69 23 71 21  0] 3\n",
      "[71 21 73 19  0] 3\n",
      "[73 19 77 15  0] 3\n",
      "[77 15 81 11  0] 3\n",
      "[81 11 85  7  0] 3\n",
      "[85  7 85  9  0] 1\n",
      "[85  9 79  5  0] 1\n",
      "[79  5 74  2  0] 3\n",
      "[74  2 70  0  0] 3\n",
      "[70  0 67  1  0] 1\n",
      "[67  1 61  5  0] 1\n",
      "[61  5 58  7  0] 1\n",
      "[58  7 52 11  0] 1\n",
      "[52 11 49 13  1] 2\n",
      "[49 13 43 17 20] 2\n",
      "[43 17 37 21 42] 2\n",
      "[37 21 32 24 60] 2\n",
      "[32 24 29 26 71] 2\n",
      "[29 26 26 28 82] 2\n",
      "[26 28 23 30 94] 2\n",
      "[ 23  30  17  34 116] 2\n",
      "[ 17  34  13  37 131] 2\n",
      "[ 13  37   7  41 132] 2\n",
      "[  7  41   2  44 132] 2\n",
      "[  2  44   0  47 132] 2\n",
      "[  0  47  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1   1  63 132] 1\n",
      "[  1  63   4  65 132] 1\n",
      "[  4  65   7  67 132] 2\n",
      "[  7  67  13  71 132] 2\n",
      "[ 13  71  16  73 132] 1\n",
      "[ 16  73  19  75 132] 2\n",
      "[ 19  75  25  79 132] 2\n",
      "[ 25  79  29  82 132] 1\n",
      "[ 29  82  32  84 132] 1\n",
      "[ 32  84  35  86 132] 2\n",
      "[ 35  86  38  88 132] 2\n",
      "[ 38  88  43  91 132] 2\n",
      "[ 43  91  49  95 132] 1\n",
      "[ 49  95  53  98 132] 1\n",
      "[ 53  98  58 101 132] 1\n",
      "[ 58 101  61 103 132] 1\n",
      "[ 61 103  65 106 132] 1\n",
      "[ 65 106  68 108 132] 1\n",
      "[ 68 108  71 110 132] 2\n",
      "[ 71 110  76 113 132] 2\n",
      "[ 76 113  80 116 132] 1\n",
      "[ 80 116  83 118 132] 2\n",
      "[ 83 118  -1  -1 128] 2\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  23   9 132] 1\n",
      "[ 23   9  27  13 132] 1\n",
      "[ 27  13  29  15 132] 1\n",
      "[ 29  15  31  17 132] 1\n",
      "[ 31  17  34  20 132] 1\n",
      "[ 34  20  36  22 132] 1\n",
      "[ 36  22  38  24 132] 1\n",
      "[ 38  24  40  26 132] 1\n",
      "[ 40  26  42  28 132] 1\n",
      "[ 42  28  45  31 132] 1\n",
      "[ 45  31  48  34 132] 1\n",
      "[ 48  34  51  37 132] 1\n",
      "[ 51  37  53  39 132] 1\n",
      "[ 53  39  56  42 132] 1\n",
      "[ 56  42  59  45 132] 1\n",
      "[ 59  45  61  47 132] 1\n",
      "[ 61  47  63  49 132] 1\n",
      "[ 63  49  66  52 132] 1\n",
      "[ 66  52  69  55 132] 1\n",
      "[ 69  55  73  59 132] 1\n",
      "[ 73  59  75  61 132] 1\n",
      "[ 75  61  77  63 132] 2\n",
      "[ 77  63  80  66 132] 2\n",
      "[ 80  66  83  69 132] 1\n",
      "[ 83  69  85  71 132] 1\n",
      "[ 85  71  -1  -1 119] 2\n",
      "ok\n",
      "[ -1  -1  -2  -2 125] 2\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  23  69 132] 1\n",
      "[ 23  69  26  66 132] 1\n",
      "[ 26  66  29  63 132] 3\n",
      "[ 29  63  31  61 132] 3\n",
      "[ 31  61  35  57 132] 3\n",
      "[ 35  57  38  54 128] 3\n",
      "[ 38  54  42  50 105] 3\n",
      "[42 50 46 46 83] 3\n",
      "[46 46 49 43 65] 3\n",
      "[49 43 53 39 43] 3\n",
      "[53 39 56 36 25] 3\n",
      "[56 36 60 32  2] 3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[60 32 64 28  0] 3\n",
      "[64 28 66 26  0] 3\n",
      "[66 26 70 22  0] 3\n",
      "[70 22 74 18  0] 3\n",
      "[74 18 77 15  0] 3\n",
      "[77 15 79 13  0] 3\n",
      "[79 13 81 11  0] 3\n",
      "[81 11 83  9  0] 3\n",
      "[83  9 86  6  0] 1\n",
      "[86  6 80  6  0] 1\n",
      "[80  6 74  2  0] 1\n",
      "[74  2 69  0  0] 3\n",
      "[69  0 66  1  0] 1\n",
      "[66  1 62  4  0] 1\n",
      "[62  4 56  8  0] 1\n",
      "[56  8 51 11  0] 1\n",
      "[51 11 45 15  4] 2\n",
      "[45 15 42 17 14] 2\n",
      "[42 17 39 19 25] 2\n",
      "[39 19 33 23 48] 2\n",
      "[33 23 27 27 71] 2\n",
      "[27 27 24 29 82] 2\n",
      "[ 24  29  20  32 100] 2\n",
      "[ 20  32  15  35 116] 2\n",
      "[ 15  35  12  37 128] 2\n",
      "[ 12  37   8  40 132] 2\n",
      "[  8  40   2  44 132] 2\n",
      "[  2  44  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1   0  67 132] 1\n",
      "[  0  67   3  69 132] 1\n",
      "[  3  69   6  71 132] 2\n",
      "[  6  71  12  75 132] 2\n",
      "[ 12  75  18  79 132] 1\n",
      "[ 18  79  23  82 132] 1\n",
      "[ 23  82  26  84 132] 1\n",
      "[ 26  84  29  86 132] 2\n",
      "[ 29  86  32  88 132] 2\n",
      "[ 32  88  35  90 132] 2\n",
      "[ 35  90  41  94 132] 2\n",
      "[ 41  94  45  97 132] 1\n",
      "[ 45  97  50 100 132] 1\n",
      "[ 50 100  56 104 132] 1\n",
      "[ 56 104  59 106 132] 1\n",
      "[ 59 106  65 110 132] 2\n",
      "[ 65 110  71 114 132] 1\n",
      "[ 71 114  75 117 132] 1\n",
      "[ 75 117  80 120 132] 2\n",
      "[ 80 120  83 122 132] 1\n",
      "[ 83 122  -1  -1 129] 2\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[-1 -1 25 67 91] 1\n",
      "[25 67 28 64 91] 1\n",
      "[28 64 32 60 79] 3\n",
      "[32 60 35 57 63] 3\n",
      "[35 57 38 54 45] 3\n",
      "[38 54 40 52 33] 3\n",
      "[40 52 43 49 16] 3\n",
      "[43 49 46 46  0] 3\n",
      "[46 46 50 42  0] 3\n",
      "[50 42 53 39  0] 3\n",
      "[53 39 55 37  0] 3\n",
      "[55 37 57 35  0] 3\n",
      "[57 35 59 33  0] 3\n",
      "[59 33 63 29  0] 3\n",
      "[63 29 66 26  0] 3\n",
      "[66 26 68 24  0] 3\n",
      "[68 24 71 21  0] 3\n",
      "[71 21 74 18  0] 3\n",
      "[74 18 76 16  0] 3\n",
      "[76 16 80 12  0] 3\n",
      "[80 12 84  8  0] 3\n",
      "[84  8 86  6  0] 1\n",
      "[86  6 83  8  0] 1\n",
      "[83  8 80  6  0] 1\n",
      "[80  6 77  4  0] 3\n",
      "[77  4 74  2  0] 3\n",
      "[74  2 69  0  0] 1\n",
      "[69  0 63  3  0] 1\n",
      "[63  3 59  6  0] 1\n",
      "[59  6 53 10  0] 1\n",
      "[53 10 50 12  1] 2\n",
      "[50 12 45 15 14] 2\n",
      "[45 15 42 17 25] 2\n",
      "[42 17 39 19 36] 2\n",
      "[39 19 36 21 42] 3\n",
      "[36 21 30 25 25] 3\n",
      "[30 25 27 27 19] 2\n",
      "[27 27 21 31 12] 3\n",
      "[21 31 18 33  1] 2\n",
      "[18 33 14 36  0] 1\n",
      "[14 36 11 38  1] 2\n",
      "[11 38  6 41 14] 2\n",
      "[ 6 41  0 45 25] 1\n",
      "[ 0 45  0 47 25] 1\n",
      "[ 0 47 -1 -1 25] 1\n",
      "[-1 -1  0 54 27] 2\n",
      "[ 0 54  2 56 33] 1\n",
      "[ 2 56  6 59 35] 1\n",
      "[ 6 59  9 61 35] 1\n",
      "[ 9 61 14 64 35] 1\n",
      "[14 64 20 68 35] 1\n",
      "[20 68 26 72 35] 1\n",
      "[26 72 32 76 35] 1\n",
      "[32 76 35 78 35] 1\n",
      "[35 78 38 80 38] 2\n",
      "[38 80 41 82 48] 2\n",
      "[41 82 45 85 66] 2\n",
      "[45 85 51 89 88] 2\n",
      "[51 89 57 93 99] 1\n",
      "[57 93 63 97 99] 1\n",
      "[ 63  97  69 101  99] 1\n",
      "[ 69 101  72 103  99] 1\n",
      "[ 72 103  75 105 102] 2\n",
      "[ 75 105  81 109 122] 2\n",
      "[ 81 109  84 111 129] 1\n",
      "[ 84 111  87 113 132] 2\n",
      "ok\n",
      "[ 87 113  -2  -2 127] 2\n",
      "ok\n",
      "[ -2  -2  -2  -2 132] 1\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  23   9 132] 1\n",
      "[ 23   9  27  13 132] 1\n",
      "[ 27  13  30  16 132] 1\n",
      "[ 30  16  34  20 132] 1\n",
      "[ 34  20  37  23 132] 1\n",
      "[ 37  23  39  25 132] 1\n",
      "[ 39  25  41  27 132] 1\n",
      "[ 41  27  44  30 132] 1\n",
      "[ 44  30  47  33 132] 1\n",
      "[ 47  33  51  37 132] 1\n",
      "[ 51  37  53  39 132] 1\n",
      "[ 53  39  57  43 132] 1\n",
      "[ 57  43  59  45 132] 1\n",
      "[ 59  45  62  48 132] 1\n",
      "[ 62  48  64  50 132] 1\n",
      "[ 64  50  67  53 132] 1\n",
      "[ 67  53  69  55 132] 1\n",
      "[ 69  55  72  58 132] 1\n",
      "[ 72  58  75  61 132] 1\n",
      "[ 75  61  78  64 132] 1\n",
      "[ 78  64  81  67 132] 1\n",
      "[ 81  67  84  70 132] 1\n",
      "[ 84  70  88  74 124] 1\n",
      "ok\n",
      "[ 88  74  -2  -2 119] 1\n",
      "ok\n",
      "[ -2  -2  -2  -2 132] 2\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  25  67 132] 1\n",
      "[ 25  67  28  64 132] 1\n",
      "[ 28  64  32  60 132] 3\n",
      "[ 32  60  34  58 132] 3\n",
      "[ 34  58  37  55 131] 3\n",
      "[ 37  55  40  52 121] 3\n",
      "[40 52 44 48 99] 3\n",
      "[44 48 47 45 81] 3\n",
      "[47 45 50 42 65] 3\n",
      "[50 42 52 40 53] 3\n",
      "[52 40 56 36 30] 3\n",
      "[56 36 58 34 19] 3\n",
      "[58 34 60 32  7] 3\n",
      "[60 32 64 28  0] 3\n",
      "[64 28 68 24  0] 3\n",
      "[68 24 70 22  0] 3\n",
      "[70 22 73 19  0] 3\n",
      "[73 19 77 15  0] 3\n",
      "[77 15 79 13  0] 3\n",
      "[79 13 82 10  0] 3\n",
      "[82 10 86  6  0] 3\n",
      "[86  6 82  7  0] 1\n",
      "[82  7 77  4  0] 1\n",
      "[77  4 71  0  0] 3\n",
      "[71  0 67  1  0] 3\n",
      "[67  1 64  3  0] 1\n",
      "[64  3 61  5  0] 1\n",
      "[61  5 58  7  0] 1\n",
      "[58  7 52 11  0] 1\n",
      "[52 11 47 14  4] 2\n",
      "[47 14 43 17 20] 2\n",
      "[43 17 37 21 42] 2\n",
      "[37 21 34 23 54] 2\n",
      "[34 23 31 25 66] 2\n",
      "[31 25 25 29 88] 2\n",
      "[ 25  29  19  33 111] 2\n",
      "[ 19  33  16  35 122] 2\n",
      "[ 16  35  10  39 132] 2\n",
      "[ 10  39   5  42 132] 2\n",
      "[  5  42   1  45 132] 2\n",
      "[  1  45   0  47 132] 2\n",
      "[  0  47  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1   1  63 132] 1\n",
      "[  1  63   4  65 132] 1\n",
      "[  4  65   7  67 132] 2\n",
      "[  7  67  10  69 132] 2\n",
      "[ 10  69  13  71 132] 2\n",
      "[ 13  71  19  75 132] 2\n",
      "[ 19  75  22  77 132] 1\n",
      "[ 22  77  25  79 132] 2\n",
      "[ 25  79  29  82 132] 2\n",
      "[ 29  82  35  86 132] 1\n",
      "[ 35  86  38  88 132] 1\n",
      "[ 38  88  43  91 132] 2\n",
      "[ 43  91  46  93 132] 1\n",
      "[ 46  93  49  95 132] 2\n",
      "[ 49  95  52  97 132] 1\n",
      "[ 52  97  58 101 132] 1\n",
      "[ 58 101  61 103 132] 1\n",
      "[ 61 103  65 106 132] 1\n",
      "[ 65 106  71 110 132] 1\n",
      "[ 71 110  74 112 132] 1\n",
      "[ 74 112  79 115 132] 2\n",
      "[ 79 115  83 118 132] 1\n",
      "[ 83 118  -1  -1 128] 2\n",
      "ok\n",
      "[ -1  -1  -2  -2 130] 1\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  22   8 132] 1\n",
      "[ 22   8  24  10 132] 1\n",
      "[ 24  10  26  12 132] 1\n",
      "[ 26  12  29  15 132] 1\n",
      "[ 29  15  33  19 132] 1\n",
      "[ 33  19  36  22 132] 1\n",
      "[ 36  22  39  25 132] 1\n",
      "[ 39  25  41  27 132] 1\n",
      "[ 41  27  45  31 132] 1\n",
      "[ 45  31  47  33 132] 1\n",
      "[ 47  33  50  36 132] 1\n",
      "[ 50  36  53  39 132] 1\n",
      "[ 53  39  55  41 132] 1\n",
      "[ 55  41  59  45 132] 1\n",
      "[ 59  45  61  47 132] 1\n",
      "[ 61  47  64  50 132] 1\n",
      "[ 64  50  68  54 132] 1\n",
      "[ 68  54  71  57 132] 1\n",
      "[ 71  57  73  59 132] 1\n",
      "[ 73  59  77  63 132] 2\n",
      "[ 77  63  81  67 132] 1\n",
      "[ 81  67  84  70 132] 1\n",
      "[ 84  70  87  73 132] 1\n",
      "ok\n",
      "[ 87  73  -2  -2 119] 1\n",
      "ok\n",
      "[ -2  -2  -2  -2 125] 2\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  22  70 132] 1\n",
      "[ 22  70  26  66 132] 1\n",
      "[ 26  66  30  62 132] 3\n",
      "[ 30  62  33  59 132] 3\n",
      "[ 33  59  37  55 130] 3\n",
      "[ 37  55  40  52 117] 3\n",
      "[40 52 44 48 95] 3\n",
      "[44 48 46 46 83] 3\n",
      "[46 46 49 43 65] 3\n",
      "[49 43 53 39 43] 3\n",
      "[53 39 56 36 25] 3\n",
      "[56 36 59 33  7] 3\n",
      "[59 33 61 31  0] 3\n",
      "[61 31 64 28  0] 3\n",
      "[64 28 68 24  0] 3\n",
      "[68 24 72 20  0] 3\n",
      "[72 20 76 16  0] 3\n",
      "[76 16 78 14  0] 3\n",
      "[78 14 82 10  0] 3\n",
      "[82 10 85  7  0] 3\n",
      "[85  7 84  9  0] 1\n",
      "[84  9 78  5  0] 1\n",
      "[78  5 74  2  0] 3\n",
      "[74  2 69  0  0] 3\n",
      "[69  0 63  3  0] 1\n",
      "[63  3 59  6  0] 1\n",
      "[59  6 54  9  0] 1\n",
      "[54  9 50 12  1] 2\n",
      "[50 12 47 14  9] 2\n",
      "[47 14 42 17 25] 2\n",
      "[42 17 38 20 42] 2\n",
      "[38 20 32 24 66] 2\n",
      "[32 24 29 26 76] 2\n",
      "[ 29  26  23  30 100] 2\n",
      "[ 23  30  18  33 116] 2\n",
      "[ 18  33  14  36 131] 2\n",
      "[ 14  36   8  40 132] 2\n",
      "[  8  40   5  42 132] 2\n",
      "[  5  42   2  44 132] 2\n",
      "[  2  44   0  47 132] 2\n",
      "[  0  47  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1   0  65 132] 1\n",
      "[  0  65   2  68 132] 1\n",
      "[  2  68   6  71 132] 2\n",
      "[  6  71   9  73 132] 1\n",
      "[  9  73  14  76 132] 2\n",
      "[ 14  76  17  78 132] 1\n",
      "[ 17  78  21  81 132] 2\n",
      "[ 21  81  27  85 132] 1\n",
      "[ 27  85  30  87 132] 1\n",
      "[ 30  87  35  90 132] 2\n",
      "[ 35  90  38  92 132] 1\n",
      "[ 38  92  44  96 132] 2\n",
      "[ 44  96  47  98 132] 1\n",
      "[ 47  98  51 101 132] 2\n",
      "[ 51 101  56 104 132] 1\n",
      "[ 56 104  60 107 132] 1\n",
      "[ 60 107  63 109 132] 1\n",
      "[ 63 109  68 112 132] 2\n",
      "[ 68 112  72 115 132] 1\n",
      "[ 72 115  78 119 132] 1\n",
      "[ 78 119  84 123 132] 1\n",
      "[ 84 123  -1  -1 129] 1\n",
      "ok\n",
      "[ -1  -1  -2  -2 132] 1\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[-1 -1 24 68 91] 1\n",
      "[24 68 28 64 91] 1\n",
      "[28 64 32 60 84] 3\n",
      "[32 60 36 56 63] 3\n",
      "[36 56 39 53 45] 3\n",
      "[39 53 41 51 33] 3\n",
      "[41 51 43 49 21] 3\n",
      "[43 49 45 47 11] 3\n",
      "[45 47 49 43  0] 3\n",
      "[49 43 52 40  0] 3\n",
      "[52 40 56 36  0] 3\n",
      "[56 36 59 33  0] 3\n",
      "[59 33 61 31  0] 3\n",
      "[61 31 63 29  0] 3\n",
      "[63 29 67 25  0] 3\n",
      "[67 25 70 22  0] 3\n",
      "[70 22 73 19  0] 3\n",
      "[73 19 75 17  0] 3\n",
      "[75 17 77 15  0] 3\n",
      "[77 15 80 12  0] 3\n",
      "[80 12 84  8  0] 3\n",
      "[84  8 84  9  0] 1\n",
      "[84  9 78  5  0] 1\n",
      "[78  5 75  3  0] 3\n",
      "[75  3 72  1  0] 1\n",
      "[72  1 69  0  0] 1\n",
      "[69  0 66  1  0] 1\n",
      "[66  1 62  4  0] 1\n",
      "[62  4 56  8  0] 1\n",
      "[56  8 51 11  0] 1\n",
      "[51 11 45 15  9] 2\n",
      "[45 15 39 19 30] 2\n",
      "[39 19 35 22 48] 2\n",
      "[35 22 32 24 60] 2\n",
      "[32 24 29 26 71] 2\n",
      "[29 26 26 28 82] 2\n",
      "[ 26  28  21  31 100] 2\n",
      "[ 21  31  17  34 116] 2\n",
      "[ 17  34  11  38 132] 2\n",
      "[ 11  38   8  40 132] 2\n",
      "[  8  40   3  43 132] 2\n",
      "[  3  43   0  45 132] 2\n",
      "[  0  45  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1   0  55 132] 1\n",
      "[  0  55   5  58 132] 1\n",
      "[  5  58   8  60 132] 1\n",
      "[  8  60  11  62 132] 2\n",
      "[ 11  62  14  64 132] 2\n",
      "[ 14  64  18  67 132] 2\n",
      "[ 18  67  23  70 132] 1\n",
      "[ 23  70  29  74 132] 1\n",
      "[ 29  74  32  76 132] 1\n",
      "[ 32  76  35  78 132] 2\n",
      "[ 35  78  41  82 132] 1\n",
      "[ 41  82  45  85 132] 1\n",
      "[ 45  85  48  87 132] 1\n",
      "[ 48  87  51  89 132] 1\n",
      "[ 51  89  54  91 132] 1\n",
      "[ 54  91  57  93 132] 1\n",
      "[ 57  93  63  97 132] 1\n",
      "[ 63  97  66  99 132] 1\n",
      "[ 66  99  71 102 132] 2\n",
      "[ 71 102  77 106 132] 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 77 106  83 110 132] 1\n",
      "[ 83 110  -1  -1 127] 1\n",
      "ok\n",
      "[ -1  -1  -2  -2 132] 1\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  24  10 132] 1\n",
      "[ 24  10  28  14 132] 1\n",
      "[ 28  14  30  16 132] 1\n",
      "[ 30  16  33  19 132] 1\n",
      "[ 33  19  36  22 132] 1\n",
      "[ 36  22  40  26 132] 1\n",
      "[ 40  26  44  30 132] 1\n",
      "[ 44  30  46  32 132] 1\n",
      "[ 46  32  48  34 132] 1\n",
      "[ 48  34  51  37 132] 1\n",
      "[ 51  37  53  39 132] 1\n",
      "[ 53  39  57  43 132] 1\n",
      "[ 57  43  60  46 132] 1\n",
      "[ 60  46  63  49 132] 1\n",
      "[ 63  49  65  51 132] 1\n",
      "[ 65  51  69  55 132] 1\n",
      "[ 69  55  72  58 132] 1\n",
      "[ 72  58  74  60 132] 1\n",
      "[ 74  60  76  62 132] 2\n",
      "[ 76  62  78  64 132] 2\n",
      "[ 78  64  80  66 132] 2\n",
      "[ 80  66  83  69 132] 2\n",
      "[ 83  69  87  73 132] 1\n",
      "[ 87  73  -1  -1 119] 1\n",
      "ok\n",
      "[ -1  -1  -2  -2 119] 2\n",
      "ok\n",
      "[ -2  -2  -2  -2 132] 1\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  22  70 132] 1\n",
      "[ 22  70  25  67 132] 1\n",
      "[ 25  67  28  64 132] 3\n",
      "[ 28  64  31  61 132] 3\n",
      "[ 31  61  35  57 132] 3\n",
      "[ 35  57  39  53 130] 3\n",
      "[ 39  53  42  50 117] 3\n",
      "[42 50 46 46 95] 3\n",
      "[46 46 50 42 71] 3\n",
      "[50 42 54 38 49] 3\n",
      "[54 38 56 36 37] 3\n",
      "[56 36 60 32 13] 3\n",
      "[60 32 64 28  0] 3\n",
      "[64 28 66 26  0] 3\n",
      "[66 26 70 22  0] 3\n",
      "[70 22 73 19  0] 3\n",
      "[73 19 77 15  0] 3\n",
      "[77 15 80 12  0] 3\n",
      "[80 12 82 10  0] 3\n",
      "[82 10 84  8  0] 3\n",
      "[84  8 83  8  0] 1\n",
      "[83  8 80  6  0] 1\n",
      "[80  6 77  4  0] 3\n",
      "[77  4 71  0  0] 3\n",
      "[71  0 67  1  0] 3\n",
      "[67  1 62  4  0] 1\n",
      "[62  4 59  6  0] 1\n",
      "[59  6 53 10  0] 1\n",
      "[53 10 49 13  4] 2\n",
      "[49 13 44 16 20] 2\n",
      "[44 16 41 18 30] 2\n",
      "[41 18 35 22 54] 2\n",
      "[35 22 29 26 76] 2\n",
      "[29 26 26 28 88] 2\n",
      "[ 26  28  20  32 111] 2\n",
      "[ 20  32  16  35 128] 2\n",
      "[ 16  35  13  37 132] 2\n",
      "[ 13  37   8  40 132] 2\n",
      "[  8  40   5  42 132] 2\n",
      "[  5  42   2  44 132] 2\n",
      "[  2  44  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1   0  62 132] 1\n",
      "[  0  62   4  65 132] 1\n",
      "[  4  65   7  67 132] 1\n",
      "[  7  67  11  70 132] 2\n",
      "[ 11  70  16  73 132] 1\n",
      "[ 16  73  22  77 132] 1\n",
      "[ 22  77  26  80 132] 1\n",
      "[ 26  80  31  83 132] 1\n",
      "[ 31  83  34  85 132] 1\n",
      "[ 34  85  37  87 132] 2\n",
      "[ 37  87  41  90 132] 2\n",
      "[ 41  90  46  93 132] 1\n",
      "[ 46  93  50  96 132] 1\n",
      "[ 50  96  55  99 132] 1\n",
      "[ 55  99  59 102 132] 1\n",
      "[ 59 102  62 104 132] 1\n",
      "[ 62 104  67 107 132] 1\n",
      "[ 67 107  70 109 132] 1\n",
      "[ 70 109  74 112 132] 2\n",
      "[ 74 112  79 115 132] 2\n",
      "[ 79 115  85 119 132] 1\n",
      "[ 85 119  88 121 130] 1\n",
      "ok\n",
      "[ 88 121  -2  -2 132] 2\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  23   9 132] 1\n",
      "[ 23   9  27  13 132] 1\n",
      "[ 27  13  31  17 132] 1\n",
      "[ 31  17  35  21 132] 1\n",
      "[ 35  21  39  25 132] 1\n",
      "[ 39  25  41  27 132] 1\n",
      "[ 41  27  43  29 132] 1\n",
      "[ 43  29  45  31 132] 1\n",
      "[ 45  31  47  33 132] 1\n",
      "[ 47  33  50  36 132] 1\n",
      "[ 50  36  53  39 132] 1\n",
      "[ 53  39  57  43 132] 1\n",
      "[ 57  43  60  46 132] 1\n",
      "[ 60  46  63  49 132] 1\n",
      "[ 63  49  66  52 132] 1\n",
      "[ 66  52  68  54 132] 1\n",
      "[ 68  54  71  57 132] 1\n",
      "[ 71  57  74  60 132] 1\n",
      "[ 74  60  77  63 132] 1\n",
      "[ 77  63  79  65 132] 1\n",
      "[ 79  65  82  68 132] 2\n",
      "[ 82  68  86  72 132] 1\n",
      "ok\n",
      "[ 86  72  -2  -2 119] 1\n",
      "ok\n",
      "[ -2  -2  -2  -2 125] 2\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  24  68 132] 1\n",
      "[ 24  68  28  64 132] 1\n",
      "[ 28  64  32  60 132] 3\n",
      "[ 32  60  34  58 132] 3\n",
      "[ 34  58  36  56 132] 3\n",
      "[ 36  56  39  53 130] 3\n",
      "[ 39  53  41  51 123] 3\n",
      "[ 41  51  44  48 105] 3\n",
      "[44 48 48 44 83] 3\n",
      "[48 44 51 41 65] 3\n",
      "[51 41 54 38 49] 3\n",
      "[54 38 58 34 25] 3\n",
      "[58 34 60 32 13] 3\n",
      "[60 32 62 30  2] 3\n",
      "[62 30 66 26  0] 3\n",
      "[66 26 70 22  0] 3\n",
      "[70 22 74 18  0] 3\n",
      "[74 18 77 15  0] 3\n",
      "[77 15 79 13  0] 3\n",
      "[79 13 81 11  0] 3\n",
      "[81 11 83  9  0] 3\n",
      "[83  9 84  9  0] 1\n",
      "[84  9 78  5  0] 1\n",
      "[78  5 74  2  0] 3\n",
      "[74  2 68  0  0] 3\n",
      "[68  0 63  3  0] 1\n",
      "[63  3 57  7  0] 1\n",
      "[57  7 54  9  0] 1\n",
      "[54  9 51 11  1] 2\n",
      "[51 11 48 13  9] 2\n",
      "[48 13 42 17 30] 2\n",
      "[42 17 39 19 42] 2\n",
      "[39 19 36 21 48] 3\n",
      "[36 21 32 24 35] 3\n",
      "[32 24 26 28 37] 2\n",
      "[26 28 23 30 48] 2\n",
      "[23 30 17 34 71] 2\n",
      "[17 34 14 36 82] 2\n",
      "[14 36 11 38 94] 2\n",
      "[ 11  38   5  42 116] 2\n",
      "[  5  42   0  46 132] 2\n",
      "[  0  46  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1   0  65 132] 1\n",
      "[  0  65   2  68 132] 1\n",
      "[  2  68   8  72 132] 2\n",
      "[  8  72  14  76 132] 1\n",
      "[ 14  76  17  78 132] 1\n",
      "[ 17  78  23  82 132] 2\n",
      "[ 23  82  26  84 132] 1\n",
      "[ 26  84  32  88 132] 2\n",
      "[ 32  88  38  92 132] 1\n",
      "[ 38  92  42  95 132] 1\n",
      "[ 42  95  47  98 132] 1\n",
      "[ 47  98  50 100 132] 1\n",
      "[ 50 100  56 104 132] 2\n",
      "[ 56 104  62 108 132] 1\n",
      "[ 62 108  68 112 132] 1\n",
      "[ 68 112  72 115 132] 1\n",
      "[ 72 115  77 118 132] 1\n",
      "[ 77 118  83 122 132] 1\n",
      "[ 83 122  87 125 132] 1\n",
      "ok\n",
      "[ 87 125  -2  -2 130] 2\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[-1 -1 24 74 91] 1\n",
      "[24 74 26 76 91] 1\n",
      "[26 76 29 79 94] 2\n",
      "[ 29  79  31  81 103] 2\n",
      "[ 31  81  35  85 126] 2\n",
      "[ 35  85  39  89 131] 1\n",
      "[ 39  89  41  91 131] 1\n",
      "[ 41  91  44  94 132] 2\n",
      "[ 44  94  47  97 132] 2\n",
      "[ 47  97  50 100 132] 2\n",
      "[ 50 100  54 104 132] 2\n",
      "[ 54 104  58 108 132] 1\n",
      "[ 58 108  62 112 132] 1\n",
      "[ 62 112  64 114 132] 1\n",
      "[ 64 114  68 118 132] 2\n",
      "[ 68 118  71 121 132] 1\n",
      "[ 71 121  74 124 132] 2\n",
      "[ 74 124  77 127 132] 2\n",
      "[ 77 127  81 131 132] 2\n",
      "[ 81 131  83 133 132] 2\n",
      "[ 83 133  85 134 132] 2\n",
      "[ 85 134  81 128 132] 2\n",
      "[ 81 128  77 122 132] 2\n",
      "[ 77 122  75 119 132] 2\n",
      "[ 75 119  71 113 132] 2\n",
      "[ 71 113  67 107 132] 2\n",
      "[ 67 107  64 103 132] 3\n",
      "[ 64 103  60  97 132] 3\n",
      "[ 60  97  56  91 130] 3\n",
      "[ 56  91  53  86 117] 3\n",
      "[ 53  86  51  83 105] 3\n",
      "[51 83 48 79 89] 3\n",
      "[48 79 44 73 65] 3\n",
      "[44 73 42 70 54] 3\n",
      "[42 70 39 65 37] 3\n",
      "[39 65 35 59 13] 3\n",
      "[35 59 31 53  0] 3\n",
      "[31 53 29 50  0] 3\n",
      "[29 50 25 44  0] 3\n",
      "[25 44 23 41  0] 3\n",
      "[23 41 19 35  0] 3\n",
      "[19 35 15 29  0] 3\n",
      "[15 29 11 23  0] 3\n",
      "[11 23  9 20  0] 3\n",
      "[ 9 20  6 16  0] 3\n",
      "[ 6 16  2 10  0] 3\n",
      "[ 2 10  0  5  0] 3\n",
      "[ 0  5 -1 -1  0] 3\n",
      "[-1 -1 -1 -1  4] 2\n",
      "[-1 -1 -1 -1 11] 1\n",
      "[-1 -1  0 11 13] 1\n",
      "[ 0 11  2 16 13] 1\n",
      "[ 2 16  4 19 13] 1\n",
      "[ 4 19  7 23  6] 3\n",
      "[ 7 23 10 28  0] 1\n",
      "[10 28 12 31  0] 1\n",
      "[12 31 15 35  0] 3\n",
      "[15 35 18 40  0] 1\n",
      "[18 40 20 43  0] 1\n",
      "[20 43 23 47  0] 2\n",
      "[23 47 26 52  1] 1\n",
      "[26 52 28 55  1] 1\n",
      "[28 55 31 59  9] 2\n",
      "[31 59 35 65 30] 2\n",
      "[35 65 38 70 40] 1\n",
      "[38 70 40 73 44] 2\n",
      "[40 73 42 76 54] 2\n",
      "[42 76 45 80 71] 2\n",
      "[45 80 49 86 94] 2\n",
      "[ 49  86  52  91 104] 1\n",
      "[ 52  91  55  95 105] 1\n",
      "[ 55  95  59 101 112] 2\n",
      "[ 59 101  61 104 122] 1\n",
      "[ 61 104  64 109 131] 2\n",
      "[ 64 109  67 113 132] 1\n",
      "[ 67 113  71 119 132] 1\n",
      "[ 71 119  75 125 132] 1\n",
      "[ 75 125  79 131 132] 1\n",
      "[ 79 131  83 137 132] 1\n",
      "[ 83 137  86 142 132] 2\n",
      "[ 86 142  80 134 132] 2\n",
      "[ 80 134  77 132 132] 2\n",
      "[ 77 132  74 130 132] 2\n",
      "[ 74 130  68 126 132] 2\n",
      "[ 68 126  65 124 132] 2\n",
      "[ 65 124  60 121 132] 2\n",
      "[ 60 121  56 118 132] 2\n",
      "[ 56 118  50 114 132] 2\n",
      "[ 50 114  47 112 132] 2\n",
      "[ 47 112  41 108 132] 2\n",
      "[ 41 108  38 106 132] 2\n",
      "[ 38 106  33 103 132] 2\n",
      "[ 33 103  29 100 132] 2\n",
      "[ 29 100  26  98 132] 2\n",
      "[ 26  98  20  94 132] 2\n",
      "[ 20  94  14  90 132] 2\n",
      "[ 14  90   9  87 132] 2\n",
      "[  9  87   5  84 132] 2\n",
      "[  5  84   2  82 132] 2\n",
      "[  2  82  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1   0  71 132] 1\n",
      "[  0  71   6  67 132] 1\n",
      "[  6  67  12  63 132] 1\n",
      "[ 12  63  17  60 132] 1\n",
      "[ 17  60  23  56 132] 3\n",
      "[ 23  56  29  52 128] 3\n",
      "[ 29  52  32  50 117] 3\n",
      "[32 50 38 46 95] 3\n",
      "[38 46 44 42 71] 3\n",
      "[44 42 50 38 49] 3\n",
      "[50 38 53 36 37] 3\n",
      "[53 36 59 32 13] 3\n",
      "[59 32 62 30  2] 3\n",
      "[62 30 66 27  0] 3\n",
      "[66 27 69 25  0] 3\n",
      "[69 25 75 21  0] 3\n",
      "[75 21 80 18  0] 3\n",
      "[80 18 84 15  0] 3\n",
      "[84 15 83 18  0] 3\n",
      "[83 18 79 24  0] 1\n",
      "[79 24 75 30  9] 2\n",
      "[75 30 73 33 20] 2\n",
      "[73 33 69 39 42] 2\n",
      "[69 39 66 44 60] 2\n",
      "[66 44 63 48 76] 2\n",
      "[ 63  48  59  54 100] 2\n",
      "[ 59  54  56  59 116] 2\n",
      "[ 56  59  53  63 131] 2\n",
      "[ 53  63  49  69 132] 2\n",
      "[ 49  69  47  72 132] 2\n",
      "[ 47  72  43  78 132] 2\n",
      "[ 43  78  41  81 132] 2\n",
      "[ 41  81  39  84 132] 2\n",
      "[ 39  84  37  87 132] 2\n",
      "[ 37  87  35  90 132] 2\n",
      "[ 35  90  32  95 132] 2\n",
      "[ 32  95  29  99 132] 2\n",
      "[ 29  99  27 102 132] 2\n",
      "[ 27 102  25 105 132] 2\n",
      "[ 25 105  21 111 132] 2\n",
      "[ 21 111  18 116 132] 2\n",
      "[ 18 116  15 120 132] 2\n",
      "[ 15 120  12 125 132] 2\n",
      "[ 12 125  10 128 132] 2\n",
      "[ 10 128   6 134 132] 2\n",
      "[  6 134   2 140 132] 2\n",
      "[  2 140   0 141 132] 2\n",
      "[  0 141  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 2\n",
      "[ -1  -1   0 125 132] 1\n",
      "[  0 125   2 119 132] 1\n",
      "[  2 119   6 113 132] 2\n",
      "[  6 113   8 110 132] 1\n",
      "[  8 110  11 105 132] 2\n",
      "[ 11 105  15  99 132] 2\n",
      "[ 15  99  18  95 132] 2\n",
      "[ 18  95  20  92 132] 2\n",
      "[ 20  92  24  86 132] 2\n",
      "[ 24  86  28  80 132] 3\n",
      "[ 28  80  31  75 132] 3\n",
      "[ 31  75  33  72 132] 3\n",
      "[ 33  72  37  66 123] 3\n",
      "[ 37  66  39  63 111] 3\n",
      "[39 63 43 57 89] 3\n",
      "[43 57 45 54 77] 3\n",
      "[45 54 49 48 54] 3\n",
      "[49 48 52 44 37] 3\n",
      "[52 44 54 41 25] 3\n",
      "[54 41 58 35  2] 3\n",
      "[58 35 60 32  0] 3\n",
      "[60 32 63 27  0] 3\n",
      "[63 27 65 24  0] 3\n",
      "[65 24 68 20  0] 3\n",
      "[68 20 70 17  0] 3\n",
      "[70 17 72 14  0] 3\n",
      "[72 14 74 11  0] 3\n",
      "[74 11 77  6  0] 3\n",
      "[77  6 80  2  0] 3\n",
      "[80  2 84  2  0] 1\n",
      "[84  2 82  4  0] 1\n",
      "[82  4 74  2  0] 1\n",
      "[74  2 66  0  0] 1\n",
      "[66  0 58  1  0] 1\n",
      "[58  1 50  3  0] 1\n",
      "[50  3 42  5  0] 1\n",
      "[42  5 34  7  9] 2\n",
      "[34  7 30  8 20] 2\n",
      "[30  8 22 10 18] 3\n",
      "[22 10 16 11  1] 3\n",
      "[16 11 12 12  1] 2\n",
      "[12 12  4 14 20] 2\n",
      "[ 4 14  0 16 28] 1\n",
      "[ 0 16 -1 -1 29] 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-1 -1  2 20 42] 2\n",
      "[ 2 20  8 21 52] 1\n",
      "[ 8 21 16 23 53] 1\n",
      "[16 23 22 25 53] 1\n",
      "[22 25 30 27 53] 1\n",
      "[30 27 38 29 53] 1\n",
      "[38 29 46 31 53] 1\n",
      "[46 31 54 33 53] 1\n",
      "[54 33 62 35 53] 1\n",
      "[62 35 68 36 53] 1\n",
      "[68 36 74 38 53] 1\n",
      "[74 38 80 39 53] 1\n",
      "[80 39 88 41 51] 1\n",
      "ok\n",
      "[88 41 -2 -2 53] 1\n",
      "[-2 -2 -1 -1 66] 2\n",
      "[-1 -1 -1 -1 75] 1\n",
      "[-1 -1 -1 -1 75] 1\n",
      "[-1 -1 23 73 75] 1\n",
      "[23 73 27 77 75] 1\n",
      "[27 77 31 81 75] 1\n",
      "[31 81 34 84 75] 1\n",
      "[34 84 37 87 78] 2\n",
      "[ 37  87  41  91 100] 2\n",
      "[ 41  91  45  95 111] 1\n",
      "[ 45  95  48  98 111] 1\n",
      "[ 48  98  51 101 113] 2\n",
      "[ 51 101  53 103 122] 2\n",
      "[ 53 103  55 105 131] 2\n",
      "[ 55 105  58 108 132] 2\n",
      "[ 58 108  61 111 132] 2\n",
      "[ 61 111  64 114 132] 2\n",
      "[ 64 114  66 116 132] 2\n",
      "[ 66 116  70 120 132] 2\n",
      "[ 70 120  72 122 132] 1\n",
      "[ 72 122  74 124 132] 2\n",
      "[ 74 124  77 127 132] 2\n",
      "[ 77 127  79 129 132] 2\n",
      "[ 79 129  82 132 132] 2\n",
      "[ 82 132  85 135 132] 2\n",
      "[ 85 135  83 131 132] 2\n",
      "[ 83 131  80 127 132] 2\n",
      "[ 80 127  77 122 132] 2\n",
      "[ 77 122  74 118 132] 2\n",
      "[ 74 118  70 112 132] 2\n",
      "[ 70 112  68 109 132] 2\n",
      "[ 68 109  64 103 132] 2\n",
      "[ 64 103  60  97 132] 3\n",
      "[ 60  97  58  94 132] 3\n",
      "[ 58  94  54  88 131] 3\n",
      "[ 54  88  51  83 123] 3\n",
      "[ 51  83  47  77 100] 3\n",
      "[47 77 43 71 77] 3\n",
      "[43 71 40 67 59] 3\n",
      "[40 67 37 62 43] 3\n",
      "[37 62 33 56 19] 3\n",
      "[33 56 30 52  2] 3\n",
      "[30 52 26 46  0] 3\n",
      "[26 46 22 40  0] 3\n",
      "[22 40 20 37  0] 3\n",
      "[20 37 16 31  0] 3\n",
      "[16 31 12 25  0] 3\n",
      "[12 25  8 19  0] 3\n",
      "[ 8 19  5 14  0] 3\n",
      "[ 5 14  2 10  0] 3\n",
      "[ 2 10  0  4  0] 3\n",
      "[ 0  4 -1 -1  0] 3\n",
      "[-1 -1 -1 -1  9] 2\n",
      "[-1 -1 -1 -1 20] 1\n",
      "[-1 -1 -1 -1 24] 1\n",
      "[-1 -1 -1 -1 25] 1\n",
      "[-1 -1  0 22 25] 1\n",
      "[ 0 22  1 26 25] 1\n",
      "[ 1 26  4 31 25] 1\n",
      "[ 4 31  7 35 25] 1\n",
      "[ 7 35 10 40 25] 1\n",
      "[10 40 14 46 25] 1\n",
      "[14 46 17 50 25] 1\n",
      "[17 50 19 53 25] 1\n",
      "[19 53 21 56 25] 3\n",
      "[21 56 25 62 31] 2\n",
      "[25 62 27 65 39] 1\n",
      "[27 65 31 71 44] 2\n",
      "[31 71 33 74 51] 1\n",
      "[33 74 37 80 66] 2\n",
      "[37 80 39 83 73] 1\n",
      "[39 83 41 86 78] 2\n",
      "[41 86 43 89 88] 2\n",
      "[ 43  89  47  95 111] 2\n",
      "[ 47  95  51 101 121] 1\n",
      "[ 51 101  53 104 121] 1\n",
      "[ 53 104  55 107 124] 2\n",
      "[ 55 107  59 113 132] 2\n",
      "[ 59 113  61 116 132] 1\n",
      "[ 61 116  64 121 132] 2\n",
      "[ 64 121  68 127 132] 2\n",
      "[ 68 127  72 133 132] 1\n",
      "[ 72 133  76 139 132] 1\n",
      "[ 76 139  80 142 132] 2\n",
      "[ 80 142  83 137 132] 2\n",
      "[ 83 137  85 134 132] 2\n",
      "[ 85 134  -1  -1 129] 2\n",
      "ok\n",
      "[ -1  -1  -2  -2 130] 1\n",
      "[ -2  -2  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[ -1  -1  -1  -1 132] 1\n",
      "[-1 -1 23 73 91] 1\n",
      "[23 73 25 75 91] 1\n",
      "[ 25  75  29  79 103] 2\n",
      "[ 29  79  33  83 113] 1\n",
      "[ 33  83  36  86 113] 1\n",
      "[ 36  86  39  89 121] 2\n",
      "[ 39  89  43  93 132] 2\n",
      "[ 43  93  45  95 132] 1\n",
      "[ 45  95  48  98 132] 2\n",
      "[ 48  98  50 100 132] 2\n",
      "[ 50 100  53 103 132] 2\n",
      "[ 53 103  56 106 132] 2\n",
      "[ 56 106  58 108 132] 2\n",
      "[ 58 108  60 110 132] 2\n",
      "[ 60 110  63 113 132] 2\n",
      "[ 63 113  67 117 132] 2\n",
      "[ 67 117  71 121 132] 1\n",
      "[ 71 121  75 125 132] 1\n",
      "[ 75 125  79 129 132] 1\n",
      "[ 79 129  83 133 132] 2\n",
      "[ 83 133  85 134 132] 2\n",
      "[ 85 134  81 128 132] 2\n",
      "[ 81 128  79 125 132] 2\n",
      "[ 79 125  76 121 132] 2\n",
      "[ 76 121  73 116 132] 2\n",
      "[ 73 116  69 110 132] 2\n",
      "[ 69 110  67 107 132] 2\n",
      "[ 67 107  63 101 132] 2\n",
      "[ 63 101  61  98 132] 3\n",
      "[ 61  98  58  94 132] 3\n",
      "[ 58  94  54  88 132] 3\n",
      "[ 54  88  50  82 123] 3\n",
      "[ 50  82  48  79 111] 3\n",
      "[48 79 45 74 95] 3\n",
      "[45 74 41 68 71] 3\n",
      "[41 68 38 64 54] 3\n",
      "[38 64 35 59 37] 3\n",
      "[35 59 32 55 19] 3\n",
      "[32 55 29 50  2] 3\n",
      "[29 50 27 47  0] 3\n",
      "[27 47 24 43  0] 3\n",
      "[24 43 21 38  0] 3\n",
      "[21 38 18 34  0] 3\n",
      "[18 34 16 31  0] 3\n",
      "[16 31 14 28  0] 3\n",
      "[14 28 11 23  0] 3\n",
      "[11 23  8 19  0] 3\n",
      "[ 8 19  5 14  0] 3\n",
      "[ 5 14  1  8  0] 3\n",
      "[1 8 0 2 0] 3\n",
      "[ 0  2 -1 -1  0] 3\n",
      "[-1 -1 -1 -1  4] 2\n",
      "[-1 -1  0  8 11] 1\n",
      "[ 0  8  0 13 13] 1\n",
      "[ 0 13  4 19 13] 1\n",
      "[ 4 19  6 22 13] 1\n",
      "[ 6 22  9 26  6] 3\n",
      "[ 9 26 12 31  0] 1\n",
      "[12 31 15 35  0] 1\n",
      "[15 35 19 41  0] 1\n",
      "[19 41 23 47  0] 1\n",
      "[23 47 26 52  0] 1\n",
      "[26 52 29 56  0] 1\n",
      "[29 56 33 62  9] 2\n",
      "[33 62 36 67 18] 1\n",
      "[36 67 40 73 26] 2\n",
      "[40 73 43 77 42] 2\n",
      "[43 77 47 83 66] 2\n",
      "[47 83 49 86 76] 1\n",
      "[49 86 51 89 80] 2\n",
      "[51 89 54 94 94] 2\n",
      "[ 54  94  57  98 111] 2\n",
      "[ 57  98  60 103 120] 1\n",
      "[ 60 103  63 107 121] 1\n",
      "[ 63 107  67 113 121] 1\n",
      "[ 67 113  70 118 121] 1\n",
      "[ 70 118  74 124 131] 2\n",
      "[ 74 124  77 128 132] 1\n",
      "[ 77 128  79 131 132] 2\n",
      "[ 79 131  83 137 132] 2\n",
      "[ 83 137  85 140 132] 2\n",
      "[ 85 140  84 137 132] 2\n",
      "[ 84 137  81 135 132] 2\n",
      "[ 81 135  75 131 132] 2\n",
      "[ 75 131  69 127 132] 2\n",
      "[ 69 127  65 124 132] 2\n",
      "[ 65 124  59 120 132] 2\n",
      "[ 59 120  56 118 132] 2\n",
      "[ 56 118  53 116 132] 2\n",
      "[ 53 116  48 113 132] 2\n",
      "[ 48 113  44 110 132] 2\n",
      "[ 44 110  39 107 132] 2\n",
      "[ 39 107  35 104 132] 2\n",
      "[ 35 104  32 102 132] 2\n",
      "[ 32 102  29 100 132] 2\n",
      "[ 29 100  24  97 132] 2\n",
      "[ 24  97  21  95 132] 2\n",
      "[ 21  95  18  93 132] 2\n",
      "[ 18  93  14  90 132] 2\n",
      "[ 14  90   8  86 132] 2\n",
      "[  8  86   5  84 132] 2\n",
      "[  5  84   0  80 132] 2\n",
      "[  0  80  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 3\n",
      "[ -1  -1   0  71 132] 1\n",
      "[  0  71   5  68 132] 1\n",
      "[  5  68   9  65 132] 1\n",
      "[  9  65  14  62 132] 2\n",
      "[ 14  62  20  58 132] 1\n",
      "[ 20  58  23  56 132] 3\n",
      "[ 23  56  26  54 132] 3\n",
      "[ 26  54  32  50 132] 3\n",
      "[ 32  50  38  46 128] 3\n",
      "[ 38  46  41  44 120] 1\n",
      "[ 41  44  45  41 110] 3\n",
      "[45 41 51 37 89] 3\n",
      "[51 37 57 33 65] 3\n",
      "[57 33 62 30 49] 3\n",
      "[62 30 66 27 31] 3\n",
      "[66 27 71 24 13] 3\n",
      "[71 24 74 22  2] 3\n",
      "[74 22 80 18  0] 3\n",
      "[80 18 84 15  0] 3\n",
      "[84 15 83 18  0] 3\n",
      "[83 18 80 23  0] 1\n",
      "[80 23 77 27  4] 2\n",
      "[77 27 75 30 14] 2\n",
      "[75 30 72 35 30] 2\n",
      "[72 35 70 38 42] 2\n",
      "[70 38 67 42 60] 2\n",
      "[67 42 64 47 76] 2\n",
      "[ 64  47  60  53 100] 2\n",
      "[ 60  53  57  57 116] 2\n",
      "[ 57  57  55  60 128] 2\n",
      "[ 55  60  53  63 132] 2\n",
      "[ 53  63  49  69 132] 2\n",
      "[ 49  69  46  74 132] 2\n",
      "[ 46  74  42  80 132] 2\n",
      "[ 42  80  40  83 132] 2\n",
      "[ 40  83  38  86 132] 2\n",
      "[ 38  86  35  90 132] 2\n",
      "[ 35  90  32  95 132] 2\n",
      "[ 32  95  29  99 132] 2\n",
      "[ 29  99  27 102 132] 2\n",
      "[ 27 102  23 108 132] 2\n",
      "[ 23 108  19 114 132] 2\n",
      "[ 19 114  15 120 132] 2\n",
      "[ 15 120  13 123 132] 2\n",
      "[ 13 123  11 126 132] 2\n",
      "[ 11 126   9 129 132] 2\n",
      "[  9 129   5 135 132] 2\n",
      "[  5 135   2 140 132] 2\n",
      "[  2 140   0 143 132] 2\n",
      "[  0 143   0 140 132] 2\n",
      "[  0 140  -1  -1 132] 2\n",
      "[ -1  -1  -1  -1 132] 2\n",
      "[ -1  -1   0 125 132] 1\n",
      "[  0 125   0 122 132] 1\n",
      "[  0 122   3 117 132] 2\n",
      "[  3 117   6 113 132] 2\n",
      "[  6 113   9 108 132] 2\n",
      "[  9 108  13 102 132] 2\n",
      "[ 13 102  15  99 132] 2\n",
      "[ 15  99  17  96 132] 2\n",
      "[ 17  96  20  92 132] 2\n",
      "[ 20  92  22  89 132] 2\n",
      "[ 22  89  24  86 132] 2\n",
      "[ 24  86  26  83 132] 2\n",
      "[ 26  83  30  77 132] 2\n",
      "[ 30  77  34  71 132] 3\n",
      "[ 34  71  38  65 132] 3\n",
      "[ 38  65  42  59 128] 3\n",
      "[ 42  59  45  54 111] 3\n",
      "[45 54 48 50 95] 3\n",
      "[48 50 51 45 77] 3\n",
      "[51 45 53 42 65] 3\n",
      "[53 42 56 38 49] 3\n",
      "[56 38 60 32 25] 3\n",
      "[60 32 63 27  7] 3\n",
      "[63 27 66 23  0] 3\n",
      "[66 23 68 20  0] 3\n",
      "[68 20 72 14  0] 3\n",
      "[72 14 75  9  0] 3\n",
      "[75  9 77  6  0] 3\n",
      "[77  6 79  3  0] 1\n",
      "[79  3 82  0  0] 1\n",
      "[82  0 86  5  0] 1\n",
      "[86  5 82  4  0] 1\n",
      "[82  4 76  2  0] 1\n",
      "[76  2 68  0  0] 1\n",
      "[68  0 62  0  0] 1\n",
      "[62  0 56  1  0] 1\n",
      "[56  1 48  3  0] 1\n",
      "[48  3 42  5  0] 1\n",
      "[42  5 34  7  9] 2\n",
      "[34  7 28  8 25] 2\n",
      "[28  8 20 10 33] 3\n",
      "[20 10 12 12 13] 3\n",
      "[12 12  4 14 15] 2\n",
      "[ 4 14 -1 -1 25] 1\n",
      "[-1 -1 -1 -1 36] 2\n",
      "[-1 -1  2 20 51] 1\n",
      "[ 2 20 10 22 53] 1\n",
      "[10 22 16 23 53] 1\n",
      "[16 23 22 25 53] 1\n",
      "[22 25 26 26 53] 1\n",
      "[26 26 34 28 53] 1\n",
      "[34 28 40 29 53] 1\n",
      "[40 29 44 30 53] 1\n",
      "[44 30 52 32 46] 3\n",
      "[52 32 56 33 38] 1\n",
      "[56 33 60 34 36] 3\n",
      "[60 34 64 35 29] 3\n",
      "[64 35 68 36 22] 1\n",
      "[68 36 74 38 20] 1\n",
      "[74 38 82 40 20] 1\n",
      "[82 40 86 41 20] 1\n",
      "ok\n",
      "[86 41 -2 -2 20] 1\n",
      "[-2 -2 -1 -1 31] 2\n",
      "[-1 -1 -1 -1 41] 1\n",
      "[-1 -1 -1 -1 41] 1\n",
      "[-1 -1 22 72 41] 1\n",
      "[22 72 26 76 41] 1\n",
      "[26 76 30 80 41] 1\n",
      "[30 80 33 83 41] 1\n",
      "[33 83 35 85 41] 2\n",
      "[35 85 39 89 60] 2\n",
      "[39 89 41 91 71] 1\n",
      "[41 91 45 95 88] 2\n",
      "[45 95 47 97 97] 1\n",
      "[ 47  97  51 101 111] 2\n",
      "[ 51 101  55 105 126] 1\n",
      "[ 55 105  58 108 127] 1\n",
      "[ 58 108  60 110 129] 2\n",
      "[ 60 110  63 113 132] 2\n",
      "[ 63 113  67 117 132] 2\n",
      "[ 67 117  70 120 132] 1\n",
      "[ 70 120  72 122 132] 2\n",
      "[ 72 122  74 124 132] 2\n",
      "[ 74 124  77 127 132] 2\n",
      "[ 77 127  80 130 132] 2\n",
      "[ 80 130  82 132 132] 2\n",
      "[ 82 132  84 134 132] 2\n",
      "[ 84 134  85 134 132] 2\n",
      "[ 85 134  83 131 132] 2\n",
      "[ 83 131  81 128 132] 2\n",
      "[ 81 128  79 125 132] 2\n",
      "[ 79 125  77 122 132] 2\n",
      "[ 77 122  73 116 132] 2\n",
      "[ 73 116  71 113 132] 2\n",
      "[ 71 113  68 109 132] 2\n",
      "[ 68 109  65 104 132] 2\n",
      "[ 65 104  63 101 132] 3\n",
      "[ 63 101  61  98 132] 2\n",
      "[ 61  98  59  95 132] 3\n",
      "[ 59  95  57  92 132] 3\n",
      "[ 57  92  53  86 131] 3\n",
      "[ 53  86  50  82 123] 3\n",
      "[ 50  82  46  76 100] 3\n",
      "[46 76 42 70 77] 3\n",
      "[42 70 39 65 59] 3\n",
      "[39 65 36 61 43] 3\n",
      "[36 61 34 58 31] 3\n",
      "[34 58 30 52  7] 3\n",
      "[30 52 28 49  0] 3\n",
      "[28 49 25 44  0] 3\n",
      "[25 44 23 41  0] 3\n",
      "[23 41 21 38  0] 3\n",
      "[21 38 18 34  0] 3\n",
      "[18 34 14 28  0] 3\n",
      "[14 28 10 22  0] 3\n",
      "[10 22  7 17  0] 3\n",
      "[ 7 17  4 13  0] 3\n",
      "[ 4 13  2 10  0] 3\n",
      "[ 2 10  0  5  0] 3\n",
      "[0 5 0 2 0] 3\n",
      "[ 0  2 -1 -1  0] 3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-1 -1 -1 -1  4] 2\n",
      "[-1 -1 -1 -1 13] 1\n",
      "[-1 -1 -1 -1 13] 1\n",
      "[-1 -1  0 25 13] 1\n",
      "[ 0 25  3 29 13] 1\n",
      "[ 3 29  7 35 13] 1\n",
      "[ 7 35 10 40 13] 1\n",
      "[10 40 12 43 13] 1\n",
      "[12 43 15 47  6] 3\n",
      "[15 47 18 52  0] 1\n",
      "[18 52 22 58  0] 1\n",
      "[22 58 24 61  0] 1\n",
      "[24 61 27 65  4] 2\n",
      "[27 65 31 71 25] 2\n",
      "[31 71 35 77 35] 1\n",
      "[35 77 38 82 35] 1\n",
      "[38 82 40 85 38] 2\n",
      "[40 85 42 88 48] 2\n",
      "[42 88 45 92 66] 2\n",
      "[45 92 47 95 76] 2\n",
      "[ 47  95  51 101 100] 2\n",
      "[ 51 101  55 107 114] 1\n",
      "[ 55 107  59 113 115] 1\n",
      "Evaluation Average Reward: 66.0\n"
     ]
    }
   ],
   "source": [
    "display.clear_output(wait=True)\n",
    "#agent.model.save_weights(\"model_{}.h5\".format(episode))\n",
    "total_reward = 0\n",
    "for i in range(TEST):\n",
    "    state = env.reset()\n",
    "    (x1,y1,p1) = preprocess(state)\n",
    "    state_shadow = np.array([x1,y1,x1,y1,p1])\n",
    "    for j in range(STEP):\n",
    "        env.render(mode='rgb_array')\n",
    "        action = agent.action(state_shadow) # direct action for test\n",
    "        next_state,reward,done,_ = env.step(action)\n",
    "        #state = preprocess(state)\n",
    "        (x2,y2,p2) = preprocess(next_state)\n",
    "        if x2 == -2:\n",
    "            reward = -50\n",
    "        else:\n",
    "            reward = 1\n",
    "        next_state_shadow = np.array([x1,y1,x2,y2,p2])\n",
    "        print(next_state_shadow,action)\n",
    "        total_reward += reward\n",
    "        state_shadow = next_state_shadow\n",
    "        x1,y1,p1 = x2,y2,p2\n",
    "        if done:\n",
    "            break\n",
    "ave_reward = total_reward/TEST\n",
    "print('Evaluation Average Reward:',ave_reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "ename": "AxisError",
     "evalue": "axis 2 is out of bounds for array of dimension 2",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mAxisError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-42-eecd68468083>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0morigin_img\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'rgb_array'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpreprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0morigin_img\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mstate_shadow\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0mtotal_reward\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m300\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<__array_function__ internals>\u001b[0m in \u001b[0;36mstack\u001b[0;34m(*args, **kwargs)\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/numpy/core/shape_base.py\u001b[0m in \u001b[0;36mstack\u001b[0;34m(arrays, axis, out)\u001b[0m\n\u001b[1;32m    426\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    427\u001b[0m     \u001b[0mresult_ndim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 428\u001b[0;31m     \u001b[0maxis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnormalize_axis_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult_ndim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    429\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    430\u001b[0m     \u001b[0msl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mslice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0maxis\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0m_nx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnewaxis\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAxisError\u001b[0m: axis 2 is out of bounds for array of dimension 2"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "from IPython import display\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "env = gym.make('Breakout-v0')\n",
    "env.reset()\n",
    "origin_img = env.render(mode='rgb_array')\n",
    "img = preprocess(origin_img)\n",
    "state_shadow = np.stack((img, img, img, img), axis=2)\n",
    "total_reward = 0 \n",
    "for _ in range(300):\n",
    "    #action = agent.action(state_shadow)\n",
    "    #print(action)\n",
    "    action = 1\n",
    "    next_state,reward,done,_  = env.step(action)\n",
    "    #next_state,reward,done,_ = env.step(env.action_space.sample())\n",
    "    total_reward += reward\n",
    "    origin_img = env.render(mode='rgb_array')\n",
    "    img = preprocess(origin_img)\n",
    "    #cv2.putText(origin_img,'there 0 error(s):',(50,150),cv2.FONT_HERSHEY_COMPLEX,6,(0,0,255),25)\n",
    "    cv2.putText(origin_img, \"re:%d:%d\" %(action,total_reward), (30, 30), cv2.FONT_HERSHEY_PLAIN, 1.0, (0, 0, 255), 0)\n",
    "    #cv2.putText(img,total_reward,(0,0))\n",
    "    plt.imshow(img)\n",
    "#    break\n",
    "    img = np.expand_dims(img,-1)\n",
    "    state_shadow = np.append( img, state_shadow[ :,:,:3 ], axis= 2 )\n",
    "    display.display(plt.gcf())\n",
    "    display.clear_output(wait=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "from IPython import display\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "env = gym.make('Breakout-v0')\n",
    "env.reset()\n",
    "origin_img = env.render(mode='rgb_array')\n",
    "img = preprocess(origin_img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f11200bd2b0>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAABHCAYAAAAunQ2/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAGeklEQVR4nO3dXahVZR7H8e9vfDvpxNg7TspopIUXdYqDFskwFZVF1E0XWRdBxbkpKAhCCYIu52aaLoZAervpZZgmm5DQzAm6M7Wsjh41K8NzqE5BURS9WP8u1rNtK2d3lrTXWc+Dvw8s9l7PXu792/L429tnvykiMDOzfP2h7QBmZvbbXNRmZplzUZuZZc5FbWaWORe1mVnmXNRmZpmrVdSSVkvaJ+mApLVNhzIzs19pqvdRS5oB7AeuAsaA7cCaiNjTfDwzM5tZ45gVwIGI+ABA0nPAjUDPop6tOTHAvP4kNLNGLLvg27Yj9LT/nbltR5h23/ENP8T3muyyOkV9NnCoa38MWPlbf2CAeazUlfUTmtm027x5V9sRerrmz4NtR5h222Jrz8vqFHUtkoaBYYABTrxHQzOzptR5MXEcWNS1vzCNHSUi1kfEUEQMzWJOv/KZmZ3w6jyj3g4slbSEqqBvBm5pNJWZNW7JpjvbjtDTMna0HSErUxZ1RByWdDewGZgBPBERuxtPZmZmQM016oh4GXi54SxmZjYJfzLRzCxzLmozs8z17e15ZlaWZbf7BbtS+Bm1mVnmXNRmZplzUZuZZc5FbWaWORe1mVnmXNRmZplzUZuZZc5FbWaWORe1mVnmXNRmZplzUZuZZc5FbWaWuVpfyiTpIPA18BNwOCKGmgxlZma/Op5vz7s8Ij5vLImZmU3KSx9mZpmrW9QBvCJpp6ThJgOZmdnR6i59rIqIcUlnAlsk7Y2I17sPSAU+DDDA3D7HNDM7cdV6Rh0R4+l0AtgArJjkmPURMRQRQ7OY09+UZmYnsCmLWtI8SSd3zgNXAyNNBzMzs0qdpY+zgA2SOsc/ExGbGk1lZmZHTFnUEfEBcOE0ZDEzs0n47XlmZplzUZuZZc5FbWaWORe1mVnmXNRmZplzUZuZZc5FbWaWOUVE/69U+gz4CDgdKPmrUZ2/PSVnB+dvW4n5/xIRZ0x2QSNFfeTKpR0l/8iA87en5Ozg/G0rPf+xvPRhZpY5F7WZWeaaLur1DV9/05y/PSVnB+dvW+n5j9LoGrWZmf1+XvowM8tcI0UtabWkfZIOSFrbxG30m6QnJE1IGukaO1XSFknvpdNT2szYi6RFkl6TtEfSbkn3pPFS8g9IekPS2yn/Q2l8iaRtaR79W9LstrP2ImmGpLckbUz7xWQHkHRQ0ruSdknakcZKmT/zJT0vaa+kUUmXlpK9rr4XtaQZwL+Aa4HlwBpJy/t9Ow14Clh9zNhaYGtELAW2pv0cHQbui4jlwCXAXenvvJT83wNXRMSFwCCwWtIlwN+BhyPiXOAL4I4WM07lHmC0a7+k7B2XR8Rg19vaSpk/jwCbIuJ8qu/OH6Wc7PVERF834FJgc9f+OmBdv2+niQ1YDIx07e8DFqTzC4B9bWeseT/+B1xVYn5gLvAmsJLqAwsz0/hR8yqnDVhIVQZXABsBlZK96z4cBE4/Ziz7+QP8CfiQ9HpbSdmPZ2ti6eNs4FDX/lgaK9FZEfFxOv8J1c+SZU3SYuAiYBsF5U9LB7uACWAL8D7wZUQcTofkPI/+CdwP/Jz2T6Oc7B0BvCJpp6ThNFbC/FkCfAY8mZaeHku/7VpC9tr8YmJNUT00Z/0WGUl/BP4L3BsRX3Vflnv+iPgpIgapnp2uAM5vOVItkq4HJiJiZ9tZfqdVEXEx1ZLlXZL+2n1hxvNnJnAx8GhEXAR8wzHLHBlnr62Joh4HFnXtL0xjJfpU0gKAdDrRcp6eJM2iKumnI+KFNFxM/o6I+BJ4jWq5YL6kzu965jqPLgNukHQQeI5q+eMRysh+RESMp9MJYAPVg2UJ82cMGIuIbWn/eariLiF7bU0U9XZgaXrVezZwM/BSA7czHV4Cbkvnb6Na+82Oqp+IfxwYjYh/dF1USv4zJM1P50+iWl8fpSrsm9JhWeaPiHURsTAiFlPN9f9HxK0UkL1D0jxJJ3fOA1cDIxQwfyLiE+CQpPPS0JXAHgrIflwaWuC/DthPtc74QNsL8TUzPwt8DPxI9Sh9B9Va41bgPeBV4NS2c/bIvorqv3bvALvSdl1B+S8A3kr5R4AH0/g5wBvAAeA/wJy2s05xP/4GbCwte8r6dtp2d/7NFjR/BoEdaf68CJxSSva6mz+ZaGaWOb+YaGaWORe1mVnmXNRmZplzUZuZZc5FbWaWORe1mVnmXNRmZplzUZuZZe4XsBMv1xDvTsIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(img[72:,5:75])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(np.where((img[70:72,5:75])!= 0)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.where(img[72:,5:75]!=0)[1][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "origin_img = env.render(mode='rgb_array')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "img = preprocess(origin_img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f10bda30da0>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD7CAYAAACscuKmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAANfklEQVR4nO3da6xlZX3H8e+vA8NYVG7SccqgQyOFkLQz6BS5mF5EWrRE+8IQqG2JIZm+sARTGwumSUPSJvhGIE01mYDWFypSlEqJlZIR0xrNcBHwwshFhDDIXECISit04N8Xe008Ducw65x9OXuf5/tJTs5ez9p7rWfNmt951l577fVPVSFp5fuV5e6ApMkw7FIjDLvUCMMuNcKwS40w7FIjhgp7knOTPJDk4SSXjapTkkYvS/2cPckq4EHgHGAncCdwYVXdP7ruSRqVQ4Z47WnAw1X1CECS64H3AAuGfXUOqzUcPsQqJb2Sn/McL9TzmW/eMGE/Dnh8zvRO4K2v9II1HM5bc/YQq5T0SrbXtgXnDRP2XpJsAbYArOFXx706SQsY5gTdE8Dxc6bXd22/pKq2VtXmqtp8KIcNsTpJwxgm7HcCJyY5Iclq4ALg5tF0S9KoLfkwvqr2Jfkr4FZgFfDJqvreyHo2pW790b0va3vw/56b97m/eejLT0b+0a9vWhHLHYf5+goL93c+l7zxrFF1Z0kW2oZJ/jsuZKj37FX1ZeDLI+qLpDHyCjqpEWM/G9+ChQ4dFzqkW6nLHZe/fPBPez93NY+NsSezzZFdaoRhlxrhYbymxkJnrOc7NF/wrDfLf9Z7WjmyS40w7FIjPIwfgXGdxZ615Q5rWvu1UjiyS41Y8s0rluKwE9bX6//+komtT2rNriv+ied/uHPe77M7skuNMOxSIyZ6gu63XvsUd5x77SRXKTXltKueWnCeI7vUCMMuNcKwS40w7FIjDLvUiIOGPcknk+xJ8t05bUcnuS3JQ93vo8bbTUnD6jOy/wtw7gFtlwHbqupEYFs3LWmKHTTsVfVfwI8PaH4P8Onu8aeBPxlxvySN2FLfs6+tqie7x7uAtSPqj6QxGfoEXQ2+SbPgt2mSbElyV5K79j794rCrk7RESw377iTrALrfexZ64tzyT8ces2qJq5M0rKWG/Wbgou7xRcCXRtMdSePS56O3zwHfBE5KsjPJxcCVwDlJHgLe0U1LmmIH/dZbVV24wKyxFlqfhtpY0rQYxS27vIJOaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVG9Lnh5PFJbk9yf5LvJbm0a7femzRD+ozs+4APVdUpwOnAB5KcgvXepJnSp9bbk1X1re7xT4EdwHFY702aKYt6z55kA3AqsJ2e9d4s/yRNh95hT/Jq4AvAB6vqJ3PnvVK9N8s/SdOhV9iTHMog6J+pqi92zb3rvUlafn3Oxge4DthRVR+bM8t6b9IMOWj5J+As4M+B7yTZX4PmIwzqu93Q1X57DDh/lB07874XRrk4qXl9ar19HcgCs8da703S6HgFndQIwy41wrBLjehzgm5ZvPGwp5a7C9KK4sguNcKwS40w7FIjDLvUCMMuNWJqz8bv3fea5e6CNEWG/56ZI7vUCMMuNcKwS40w7FIjpvYE3ZrsW+4uSCuKI7vUCMMuNWJqD+OPWPXccndBWlH63HByTZI7ktzXlX+6oms/Icn2JA8n+XyS1ePvrqSl6nMY/zzw9qraCGwCzk1yOvBR4KqqehPwDHDx+LopaVh9yj9VVf2smzy0+yng7cCNXbvln6Qp17dIxKruNtJ7gNuAHwDPVtX+z8d2Mqj/Nt9rLf8kTYFeJ+iq6kVgU5IjgZuAk/uuoKq2AlsBNm9cM2+JqPncsOt3+j5VWvH+4rX/MfQyFvXRW1U9C9wOnAEcmWT/H4v1wBND90bS2PQ5G39sN6KT5FXAOQzKNt8OvLd7muWfpCnX5zB+HfDpJKsY/HG4oapuSXI/cH2SfwDuYVAPTtKU6lP+6dsMarIf2P4IcNo4OiVp9LxcVmrE1F4u+9tHeL5PGiVHdqkRhl1qhGGXGmHYpUZM7Qm6r19x+nJ3QZoeH//20ItwZJcaYdilRkztYfyr/u2O5e6CND0+PvwiHNmlRhh2qRGGXWqEYZcaYdilRhh2qRGGXWqEYZca0Tvs3b3j70lySzdt+SdphixmZL+UwV1l97P8kzRD+laEWQ/8MXBtNx0s/yTNlL4j+9XAh4GXuuljsPyTNFP6FIk4D9hTVXcvZQVVtbWqNlfV5mOPWbWURUgagT7fejsLeHeSdwFrgNcC19CVf+pGd8s/SVOuT8nmy6tqfVVtAC4AvlpV78PyT9JMGeZz9r8F/jrJwwzew1v+SZpii7p5RVV9Dfha99jyT9IM8Qo6qRGGXWqEYZcaYdilRhh2qRGGXWqEYZcaYdilRhh2qRGGXWqEYZcaYdilRhh2qRGGXWqEYZcaYdilRhh2qRGGXWpEr9tSJXkU+CnwIrCvqjYnORr4PLABeBQ4v6qeGU83pen2xGVnvqztuCu/sQw9WdhiRvY/qKpNVbW5m74M2FZVJwLbumlJU2qYw/j3MCj7BJZ/UuNWPf/yn2nTN+wF/GeSu5Ns6drWVtWT3eNdwNqR907SyPS9lfTbquqJJL8G3Jbk+3NnVlUlqfle2P1x2ALwhuMWdedqSSPUK31V9UT3e0+SmxjcL353knVV9WSSdcCeBV67FdgKsHnjmnn/IEiz7vVXTdfJuPn0Kex4eJLX7H8M/CHwXeBmBmWfwPJP0tTrM7KvBW4alGTnEOCzVfWVJHcCNyS5GHgMOH983ZQ0rIOGvSvztHGe9qeBs8fRKUmj5xV0UiMMu9QIwy41wrBLjTDsUiMMu9QIwy41wrBLjTDsUiMMu9QIwy41wrBLjTDsUiMMu9QIwy41wrBLjTDsUiMMu9SIXmFPcmSSG5N8P8mOJGckOTrJbUke6n4fNe7OSlq6viP7NcBXqupkBvej24Hln6SZ0udW0kcAvwtcB1BVL1TVs1j+SZopfUb2E4C9wKeS3JPk2u7+8ZZ/kmZIn7AfArwZ+ERVnQo8xwGH7FVVDOrBvUySLUnuSnLX3qdfHLa/kpaoT9h3Ajurans3fSOD8O/uyj5xsPJPVbW5qjYfe8yqUfRZ0hIcNOxVtQt4PMlJXdPZwP1Y/kmaKX3Lql4CfCbJauAR4P0M/lBY/kmaEX2ruN4LbJ5nluWfpBnhFXRSIwy71AjDLjXCsEuNMOxSIwy71AjDLjXCsEuNMOxSIwy71AjDLjXCsEuNMOxSIwy71AjDLjXCsEuNMOxSIwy71Ig+RSJOSnLvnJ+fJPmg5Z+k2dLn7rIPVNWmqtoEvAX4H+AmLP8kzZTFHsafDfygqh7D8k/STFls2C8APtc9tvyTNEN6h727Z/y7gX89cJ7ln6Tpt5iR/Z3At6pqdzdt+Sdphiwm7Bfyi0N4sPyTNFN6hb0r0XwO8MU5zVcC5yR5CHhHNy1pSvUt//QccMwBbU+zyPJPu19czdXPbFjMS6R5nXnfC2NZ7jc2rh7Lcod1xd5Tej3vR/v2LjjPK+ikRhh2qRF9SzZLU+Wz//57Y1nuBr45luVOA0d2qRGGXWrERA/jX3jpEB79+et6PnvfWPui2bbh71bu4fZ8+n5K8LNaePx2ZJcaMdGR/X93wI63OGJLy8GRXWqEYZcaYdilRhh2qRGGXWqEYZcaYdilRhh2qRGGXWqEYZcaYdilRhh2qRGGXWpEBsVcJrSyZC/wHPDUxFY6Wa9jZW6b2zU73lhVx843Y6JhB0hyV1VtnuhKJ2SlbpvbtTJ4GC81wrBLjViOsG9dhnVOykrdNrdrBZj4e3ZJy8PDeKkREw17knOTPJDk4SSXTXLdo5Tk+CS3J7k/yfeSXNq1H53ktiQPdb+PWu6+LkWSVUnuSXJLN31Cku3dfvt8kumsfngQSY5McmOS7yfZkeSMlbLP+phY2JOsAv4ZeCdwCnBhkn6lKafPPuBDVXUKcDrwgW5bLgO2VdWJwLZuehZdCuyYM/1R4KqqehPwDHDxsvRqeNcAX6mqk4GNDLZxpeyzg6uqifwAZwC3zpm+HLh8Uusf87Z9iUH9+geAdV3bOuCB5e7bErZlPYP/9G8HbgHC4MKTQ+bbj7PyAxwB/JDuPNWc9pnfZ31/JnkYfxzw+JzpnV3bTEuyATgV2A6sraonu1m7gLXL1K1hXA18GHipmz4GeLaq9t/wf1b32wnAXuBT3VuUa5MczsrYZ714gm4ISV4NfAH4YFX9ZO68GgwVM/VRR5LzgD1Vdfdy92UMDgHeDHyiqk5lcNn2Lx2yz+I+W4xJhv0J4Pg50+u7tpmU5FAGQf9MVX2xa96dZF03fx2wZ7n6t0RnAe9O8ihwPYND+WuAI5Psrx40q/ttJ7CzqrZ30zcyCP+s77PeJhn2O4ETuzO7q4ELgJsnuP6RSRLgOmBHVX1szqybgYu6xxcxeC8/M6rq8qpaX1UbGOyfr1bV+4Dbgfd2T5u57QKoql3A40lO6prOBu5nxvfZYkz6W2/vYvCecBXwyar6x4mtfISSvA34b+A7/OK97UcYvG+/AXgD8BhwflX9eFk6OaQkvw/8TVWdl+Q3GIz0RwP3AH9WVc8vZ/+WIskm4FpgNfAI8H4GA96K2GcH4xV0UiM8QSc1wrBLjTDsUiMMu9QIwy41wrBLjTDsUiMMu9SI/wc3NnA36vPDXgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}