PolicyGradientKlonSolve.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on\n",
    "\n",
    "- https://github.com/pytorch/examples/blob/87d9a1e930b5b813/reinforcement_learning/reinforce.py\n",
    "- https://medium.com/@ts1829/policy-gradient-reinforcement-learning-in-pytorch-df1383ea0baf [Notebook](https://nbviewer.jupyter.org/urls/gist.githubusercontent.com/ts1829/ebbe2cf946bf36951b724818c52e36b9/raw/4da449bffe9835e201f2fb34f381fbb53568d1ca/Policy%20Gradient%20with%20Cartpole%20and%20PyTorch%20%28Medium%20Version%29.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from itertools import count\n",
    "import itertools\n",
    "from collections import namedtuple, defaultdict\n",
    "from tqdm import tqdm, trange\n",
    "import os\n",
    "import json\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "from torch.distributions import Categorical\n",
    "%matplotlib inline\n",
    "\n",
    "import random\n",
    "from tuplestate import *\n",
    "from gamestate import *\n",
    "from benchmarking import *\n",
    "from vectorize import *\n",
    "random.seed(0)\n",
    "# print(to_pretty_string(klonstate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All seeds\n",
      "Solved-Min      7,876\n",
      "Solved            320\n",
      "Impossible      1,282\n",
      "Unknown           522\n",
      "---------------------\n",
      "Total          10,000\n"
     ]
    }
   ],
   "source": [
    "all_solutions = os.listdir('./bench/shootme/')\n",
    "\n",
    "def solve_state(ret):\n",
    "    lines = ret.splitlines()\n",
    "    result = lines[15]\n",
    "    if result.startswith('Minimal solution'):\n",
    "        return \"Solved-Min\"\n",
    "    elif result.startswith(\"Solved\"):\n",
    "        return \"Solved\"\n",
    "    elif result.startswith('Impossible'):\n",
    "        return \"Impossible\"\n",
    "    elif result.startswith('Unknown'):\n",
    "        return \"Unknown\"\n",
    "\n",
    "def clf_seeds(seedlist):\n",
    "    results = defaultdict(set)\n",
    "    for seed in seedlist:\n",
    "        with open(f\"./bench/shootme/{seed}\") as f:\n",
    "            ret = f.read()\n",
    "            result = solve_state(ret)\n",
    "            results[result].add(seed)\n",
    "    return results\n",
    "\n",
    "def clf_summary(seedlist):\n",
    "    results = clf_seeds(seedlist)\n",
    "    states = ['Solved-Min', 'Solved', 'Impossible', 'Unknown']\n",
    "    for clfstate in states:\n",
    "        seeds = results[clfstate]\n",
    "        print(f\"{clfstate:12} {len(seeds):8,}\")\n",
    "        total = sum(len(s) for s in results.values())\n",
    "    print(('-'*12) + '-' + ('-'*8))\n",
    "    print(f\"{'Total':12} {total:8,}\") \n",
    "          \n",
    "def get_state(ret):\n",
    "    deck_json = convert_shootme_to_solvitaire_json(ret)\n",
    "    return init_from_solvitaire(deck_json)\n",
    "\n",
    "def map_seeds_to_states(seed_seq):\n",
    "    for seed in seed_seq:\n",
    "        with open(f\"bench/shootme/{seed}\") as f:\n",
    "            ret = f.read()\n",
    "            state = get_state(ret)\n",
    "            yield seed, state\n",
    "\n",
    "print(\"All seeds\")\n",
    "clf_summary(all_solutions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- In features: game state vector of size $233 \\times 104$\n",
    "\n",
    "- Out features: legal moves vector of size $623$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "IN = 233*104\n",
    "OUT = 623\n",
    "\n",
    "Args = namedtuple('Args', 'gamma lr seed render log_interval')\n",
    "args = Args(\n",
    "    gamma=0.5,\n",
    "    lr=5e-2,\n",
    "    seed=1,\n",
    "    render=False,\n",
    "    log_interval=20)\n",
    "\n",
    "\n",
    "class Policy(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Policy, self).__init__()\n",
    "        self.affine1 = nn.Linear(IN, 800)\n",
    "        self.dropout = nn.Dropout(p=0.6)\n",
    "        self.affine2 = nn.Linear(800, OUT)\n",
    "        self.saved_log_probs = []\n",
    "        self.rewards = []\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.affine1(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.dropout(x)\n",
    "        action_scores = self.affine2(x)\n",
    "        return F.softmax(action_scores, dim=1)\n",
    "\n",
    "def select_action(klonstate):\n",
    "    state_vec = state_to_vec(klonstate)\n",
    "    movefilter = vector_legal_moves(klonstate)\n",
    "    torch_state_vec = torch.from_numpy(state_vec).float().reshape(-1).unsqueeze(0)\n",
    "    torch_filter = torch.from_numpy(movefilter.astype(np.float32)).unsqueeze(0)\n",
    "    probs = policy(torch_state_vec) * torch_filter\n",
    "    if (probs == 0).all():\n",
    "        torch_filter.requires_grad_()\n",
    "        # sample all legal moves with uniform probability\n",
    "        m = Categorical(torch_filter)\n",
    "    else:\n",
    "        # :attr:`probs` will be normalized to sum to 1\n",
    "        m = Categorical(probs)\n",
    "    action = m.sample()\n",
    "    log_prob = m.log_prob(action)\n",
    "    policy.saved_log_probs.append(log_prob)\n",
    "    return action.item()\n",
    "\n",
    "def finish_episode():\n",
    "    R = 0\n",
    "    policy_loss = []\n",
    "    returns = []\n",
    "    for r in policy.rewards[::-1]:\n",
    "        R = r + args.gamma * R\n",
    "        returns.insert(0, R)\n",
    "    returns = torch.tensor(returns)\n",
    "    returns = (returns - returns.mean()) / (returns.std() + eps)\n",
    "    for log_prob, R in zip(policy.saved_log_probs, returns):\n",
    "        policy_loss.append(-log_prob * R)\n",
    "    optimizer.zero_grad()\n",
    "    policy_loss = torch.cat(policy_loss).sum()\n",
    "    policy_loss.backward()\n",
    "    optimizer.step()\n",
    "    del policy.rewards[:]\n",
    "    del policy.saved_log_probs[:]\n",
    "    \n",
    "def step(curr_state, move_code):\n",
    "    new_state = play_move(curr_state, move_code)\n",
    "    reward = 0  \n",
    "    if state_is_win(new_state) or all_cards_faceup(new_state):\n",
    "        reward = 1\n",
    "    if not state_is_legal(new_state):\n",
    "        print('\\n got illegal state by playing move', move_code)\n",
    "        print('prev state')\n",
    "        print(to_pretty_string(curr_state))\n",
    "        print('\\nnew state')\n",
    "        print(to_pretty_string(new_state))\n",
    "        assert state_is_legal(new_state), 'got illegal state'\n",
    "    return new_state, reward\n",
    "\n",
    "def score_state(klonstate):\n",
    "    fnds = [klonstate.foundation1, klonstate.foundation2, \n",
    "            klonstate.foundation3, klonstate.foundation4]\n",
    "    return sum(map(len, fnds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  10   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  19   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  62   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  91   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps   5   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  79   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  70   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  67   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  82   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  69   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  63   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  33   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  63   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  65   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  58   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  19   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  65   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  58   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  62   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  71   \n",
      "ep reward -1.00   steps  67   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  66   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  33   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  58   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  61   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  31   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  58   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  42   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  65   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  62   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  68   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  61   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  59   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  61   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  30   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  62   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  62   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  59   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  68   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  29   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  59   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  63   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  62   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  22   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  61   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  25   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  67   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  58   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  65   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  61   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  59   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  64   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  63   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  64   \n",
      "ep reward -1.00   steps  59   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  15   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  58   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  78   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  68   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  59   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  58   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  40   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  58   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  68   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  80   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  58   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  96   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  63   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  46   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  69   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  89   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  72   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps  65   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  71   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  64   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  62   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  58   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  59   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  63   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  62   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps   9   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  70   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  59   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  59   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  63   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  71   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  65   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  64   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps   4   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  64   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  61   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  37   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  64   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  65   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  66   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  65   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  44   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  86   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  65   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  59   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  61   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  61   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  59   \n",
      "ep reward -1.00   steps  59   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  68   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  62   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  44   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  61   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  58   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  68   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  68   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  54   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  58   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  62   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  62   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  73   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  56   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  63   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps   5   \n",
      "ep reward -1.00   steps   8   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  48   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  12   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps   6   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  69   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  57   \n",
      "ep reward -1.00   steps  61   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  70   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  61   \n",
      "ep reward -1.00   steps  26   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  55   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  10   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  70   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  52   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps   9   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  46   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  49   \n",
      "ep reward -1.00   steps  24   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  27   \n",
      "ep reward -1.00   steps  17   \n",
      "ep reward -1.00   steps  33   \n",
      "ep reward -1.00   steps  60   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  75   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  45   \n",
      "ep reward -1.00   steps   7   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  51   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  31   \n",
      "ep reward -1.00   steps   3   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  63   \n",
      "ep reward -1.00   steps  35   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  20   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  39   \n",
      "ep reward -1.00   steps  23   \n",
      "ep reward -1.00   steps  32   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  13   \n",
      "ep reward -1.00   steps  22   \n",
      "ep reward -1.00   steps  29   \n",
      "ep reward -1.00   steps  19   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  15   \n",
      "ep reward -1.00   steps  41   \n",
      "ep reward -1.00   steps  12   \n",
      "ep reward -1.00   steps  30   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  37   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  25   \n",
      "ep reward -1.00   steps  28   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  44   \n",
      "ep reward -1.00   steps  43   \n",
      "ep reward -1.00   steps  16   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  53   \n",
      "ep reward -1.00   steps  38   \n",
      "ep reward -1.00   steps  42   \n",
      "ep reward -1.00   steps  40   \n",
      "ep reward -1.00   steps  21   \n",
      "ep reward -1.00   steps  36   \n",
      "ep reward -1.00   steps  34   \n",
      "ep reward -1.00   steps  14   \n",
      "ep reward -1.00   steps  18   \n",
      "ep reward -1.00   steps  47   \n",
      "ep reward -1.00   steps  50   \n",
      "ep reward -1.00   steps  11   \n",
      "ep reward -1.00   steps  58   \n",
      "ep reward -1.00   steps  64   \n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "invalid multinomial distribution (encountering probability entry < 0)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-174-ac0e961a8e97>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0mvisited\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m         \u001b[0maction_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mselect_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mklonstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m         \u001b[0mmove_code\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mall_moves\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maction_idx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0mklonstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mklonstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmove_code\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-172-91b25b7f7ca7>\u001b[0m in \u001b[0;36mselect_action\u001b[0;34m(klonstate)\u001b[0m\n\u001b[1;32m     40\u001b[0m         \u001b[0;31m# :attr:`probs` will be normalized to sum to 1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m         \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCategorical\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m     \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m     \u001b[0mlog_prob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_prob\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m     \u001b[0mpolicy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msaved_log_probs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlog_prob\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/ada/projects/klonsolve/klonsolve/lib/python3.6/site-packages/torch/distributions/categorical.py\u001b[0m in \u001b[0;36msample\u001b[0;34m(self, sample_shape)\u001b[0m\n\u001b[1;32m    105\u001b[0m         \u001b[0mprobs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprobs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m         \u001b[0mprobs_2d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprobs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_events\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 107\u001b[0;31m         \u001b[0msample_2d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmultinomial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprobs_2d\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    108\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0msample_2d\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: invalid multinomial distribution (encountering probability entry < 0)"
     ]
    }
   ],
   "source": [
    "# policy = Policy()\n",
    "# optimizer = optim.SGD(policy.parameters(), lr=1e-2)\n",
    "# eps = np.finfo(np.float32).eps.item()\n",
    "\n",
    "# random.seed(0)\n",
    "# train_seeds = random.sample(all_solutions, k=100)\n",
    "# print('Training game set')\n",
    "# clf_summary(train_seeds)\n",
    "\n",
    "training_games = map_seeds_to_states(all_solutions)\n",
    "\n",
    "for i, env in enumerate(training_games):\n",
    "    seed, klonstate = env\n",
    "#     print(seed)\n",
    "    ep_reward = 0\n",
    "    done = False\n",
    "    path = []\n",
    "    visited = set()\n",
    "    for t in range(1, 1000):\n",
    "        action_idx = select_action(klonstate)\n",
    "        move_code = all_moves[action_idx]\n",
    "        klonstate, reward = step(klonstate, move_code)\n",
    "        path.append(move_code)\n",
    "        if klonstate in visited:\n",
    "            reward = -1\n",
    "        else:\n",
    "            visited.add(klonstate)\n",
    "        done = reward != 0\n",
    "#         if reward > 0:\n",
    "#             # if it didnt hit a cycle then reward = state score\n",
    "#             reward = score_state(klonstate)\n",
    "#         elif reward < 0: # hit cycle\n",
    "#             # few steps hitting a cycle gets large negative reward\n",
    "#             # many steps hitting a cycle gets a smaller negative reward\n",
    "#             reward = 10/(t * reward)\n",
    "        policy.rewards.append(reward)\n",
    "        ep_reward += reward\n",
    "        if done:\n",
    "#             print(f'{seed:11} done after {t:4} steps {ep_reward:.2f}')\n",
    "            break\n",
    "    print(f\"{i:5} ep reward {ep_reward:.2f}   steps {t:3}\")#, end=' ')\n",
    "#     print('final score', score_state(klonstate))\n",
    "#     print('path', path)\n",
    "#     print('final state')\n",
    "#     print(to_pretty_string(klonstate))\n",
    "    \n",
    "    finish_episode()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save model to checkpoint!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stock: \n",
      "Waste: KD QS\n",
      "Fnd C: AC 2C 3C 4C 5C 6C 7C 8C\n",
      "Fnd D: AD 2D 3D 4D 5D 6D 7D 8D\n",
      "Fnd S: AS 2S 3S 4S 5S 6S 7S 8S\n",
      "Fnd H: AH 2H 3H 4H 5H 6H 7H 8H\n",
      "Tab 1: \n",
      "Tab 2: KH QC JD TC 9H\n",
      "Tab 3: KS QD JS TH 9S\n",
      "Tab 4: JH TS 9D\n",
      "Tab 5: \n",
      "Tab 6: KC QH JC TD 9C\n",
      "Tab 7: \n",
      "\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "invalid multinomial distribution (encountering probability entry < 0)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-175-4d35c274d5ce>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m500\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mvisited\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mklonstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mselect_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mklonstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;31m#     print(f'iteration {i}  action {all_moves[action]}')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0mklonstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mklonstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-172-91b25b7f7ca7>\u001b[0m in \u001b[0;36mselect_action\u001b[0;34m(klonstate)\u001b[0m\n\u001b[1;32m     40\u001b[0m         \u001b[0;31m# :attr:`probs` will be normalized to sum to 1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m         \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCategorical\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m     \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m     \u001b[0mlog_prob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_prob\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m     \u001b[0mpolicy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msaved_log_probs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlog_prob\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/ada/projects/klonsolve/klonsolve/lib/python3.6/site-packages/torch/distributions/categorical.py\u001b[0m in \u001b[0;36msample\u001b[0;34m(self, sample_shape)\u001b[0m\n\u001b[1;32m    105\u001b[0m         \u001b[0mprobs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprobs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m         \u001b[0mprobs_2d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprobs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_events\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 107\u001b[0;31m         \u001b[0msample_2d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmultinomial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprobs_2d\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    108\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0msample_2d\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: invalid multinomial distribution (encountering probability entry < 0)"
     ]
    }
   ],
   "source": [
    "# klonstate = random_solved_endgame(20)\n",
    "# print(to_pretty_string(klonstate))\n",
    "# print()\n",
    "\n",
    "# done = False\n",
    "# visited = set()\n",
    "# for i in range(500):\n",
    "#     visited.add(klonstate)\n",
    "#     action = select_action(klonstate)\n",
    "# #     print(f'iteration {i}  action {all_moves[action]}')\n",
    "#     klonstate, reward = step(klonstate, action)\n",
    "#     done = reward > 0\n",
    "#     if done:\n",
    "#         break\n",
    "\n",
    "# print()\n",
    "# if done:\n",
    "#     print(\"Solved!\")\n",
    "# else:\n",
    "#     print(f'Not solved after {i+1} iterations')\n",
    "    \n",
    "# print(to_pretty_string(klonstate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "         nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan]],\n",
       "       grad_fn=<SoftmaxBackward>)"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "policy(torch_state_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "invalid multinomial distribution (encountering probability entry < 0)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-178-2d6144cf3a8e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0;31m# :attr:`probs` will be normalized to sum to 1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCategorical\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m \u001b[0mlog_prob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_prob\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/ada/projects/klonsolve/klonsolve/lib/python3.6/site-packages/torch/distributions/categorical.py\u001b[0m in \u001b[0;36msample\u001b[0;34m(self, sample_shape)\u001b[0m\n\u001b[1;32m    105\u001b[0m         \u001b[0mprobs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprobs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m         \u001b[0mprobs_2d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprobs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_events\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 107\u001b[0;31m         \u001b[0msample_2d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmultinomial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprobs_2d\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    108\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0msample_2d\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: invalid multinomial distribution (encountering probability entry < 0)"
     ]
    }
   ],
   "source": [
    "state_vec = state_to_vec(klonstate)\n",
    "movefilter = vector_legal_moves(klonstate)\n",
    "torch_state_vec = torch.from_numpy(state_vec).float().reshape(-1).unsqueeze(0)\n",
    "torch_filter = torch.from_numpy(movefilter.astype(np.float32)).unsqueeze(0)\n",
    "probs = policy(torch_state_vec) * torch_filter\n",
    "if (probs == 0).all():\n",
    "    torch_filter.requires_grad_()\n",
    "    # sample all legal moves with uniform probability\n",
    "    m = Categorical(torch_filter)\n",
    "else:\n",
    "    # :attr:`probs` will be normalized to sum to 1\n",
    "    m = Categorical(probs)\n",
    "action = m.sample()\n",
    "log_prob = m.log_prob(action)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "klonsolve",
   "language": "python",
   "name": "klonsolve"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}