heuristic_markovian_agent.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython import display\n",
    "\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import gym\n",
    "\n",
    "env_name = \"Skiing-v0\"\n",
    "env = gym.make(env_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def render(x, step=0):\n",
    "  display.clear_output(wait=True)\n",
    "\n",
    "  plt.figure(figsize=(6, 6))\n",
    "  plt.clf()\n",
    "  plt.axis(\"off\")\n",
    "  plt.title(\"step: %d\" % step)\n",
    "  plt.imshow(x, cmap=plt.cm.gray)\n",
    "  plt.pause(0.001)   # pause for plots to update"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Objects can be distinquished by RGB codes.\n",
    "# Player: [214, 92, 92]\n",
    "# Flags (blue): [66, 72, 200]\n",
    "# Flags (red): [184, 50, 50]\n",
    "\n",
    "def get_pos_player(observe):\n",
    "  ids = np.where(np.sum(observe == [214, 92, 92], -1) == 3)\n",
    "  return ids[0].mean(), ids[1].mean()\n",
    "\n",
    "def get_pos_flags(observe):\n",
    "  if np.any(np.sum(observe == [184, 50, 50], -1) == 3):\n",
    "    ids = np.where(np.sum(observe == [184, 50, 50], -1) == 3)\n",
    "    return ids[0].mean(), ids[1].mean()\n",
    "  else:\n",
    "    base = 0\n",
    "    ids = np.where(np.sum(observe == [66, 72, 200], -1) == 3)\n",
    "    return ids[0].mean() + base, ids[1].mean()\n",
    "\n",
    "def get_speed(observe, observe_old):\n",
    "  # As the vertical location of the player is not changed, \n",
    "  # I estimate the vertical speed by measuring how much frames are shifted up.\n",
    "  min_val = np.inf\n",
    "  min_idx = 0\n",
    "  for k in range(0, 7):\n",
    "    val = np.sum(np.abs(observe[54:-52,8:152] - observe_old[54+k:-52+k,8:152]))\n",
    "    if min_val > val:\n",
    "      min_idx = k\n",
    "      min_val = val\n",
    "  return min_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "observe = env.reset()\n",
    "step = 0\n",
    "done = False\n",
    "h, w, c = observe.shape\n",
    "# states\n",
    "r_a, c_a = get_pos_player(observe)\n",
    "r_f, c_f = get_pos_flags(observe)\n",
    "r_a_old, c_a_old = r_a, c_a\n",
    "observe_old = observe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 432x432 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQYAAAFkCAYAAADCCZVtAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAj2ElEQVR4nO3deXwVVZ738c+5S25WyErYZBEFBeL2YKOtiOPC6Dh2j3u7OzI67fLqtlXQ0R7bUdoNZuinwe7nEbVxaWbE5dFnWkRtBaSVILjRssiOEtYsZE/uduaPwDWhEkjgkrr35vt+vfIyt+pU3V8i95tzqk5VGWstIiKtedwuQEQSj4JBRBwUDCLioGAQEQcFg4g4KBhExEHBICIOCoYkZYx52Bjzcje+X91+XxFjzIxW6680xqw2xtQaY1YZY/5hv+1/YYzZYYypMcY8b4wJdFft0nUKBukUa232vi+gL9AIvApgjBkAvAzcDfQCJgFzjDF99q7/W+B+4FxgMHA08G/d/kNIpykYEpwx5j5jTNnev8TfGGPONcZcADwAXLX3r/dXe9v2NsY8Z4zZvnebKcYY7951NxljPjbGzDTGVBtj1hhjzj3Esi4DdgGL974eCOyx1r5jW7wN1APD9q6/EXjOWrvSWlsFPArcdIjvLd1AwZDAjDEjgDuBU621OcDfAputtfOBx4BX9v4VP3HvJrOBMHAMcDIwAfinVrscC2wACoFfAW8YY/L3vtf9xpg/dbK0G4EX7ffz6ZcDq40xPzLGePcOI5qBFXvXjwK+arX9V0CxMaagk+8n3czndgFyQBEgAIw0xuy21m7uqKExphj4OyDXWtsI1BtjpgO3Av93b7NdwG/2fqBfMcbcA1wEvGStfaIzBRljBgPjgYn7lllrI8aYF4E5QDoQBK6w1tbvbZINVLfazb7vc4CKzryvdC/1GBKYtXY9cBfwMLDLGPNfxpj+HTQfDPiB7caYPcaYPbQEQp9Wbcps26vmtgAd7a8j1wN/sdZu2rfAGHMe8BRwNpBGS3A8a4w5aW+TOlqOPeyz7/vaLr63dBMFQ4Kz1s6x1p5JywffAk/uW7Vf0+9o6b4XWmtz9371staOatVmgDHGtHo9CNjWxZJuAF7Yb9lJwEfW2uXW2qi1dhmwFDhv7/qVwImt2p8I7LTWqreQoBQMCcwYM8IYc87eU3tNtJwJiO5dvRMYYozxAFhrtwPvAf9ujOlljPEYY4YZY8a32mUf4GfGGL8x5grgeGBeF+r5ITCAvWcjWlkGjNvXQzDGnAyM4/tjDC8CE40xI40xucAvaTkeIglKwZDYAsATQDmwg5YP9r/sXbfvw1lhjPl87/c30NKVXwVUAa8B/Vrtbylw7N79/Rq4fN9fbWPMA8aYdw5Sz43AG9baNkMAa+0iWoY7rxljaoHXgceste/tXT+flqHGAuBbWoYwv+rcr0DcYHSjlp7BGHMT8E97hyUiB6Qeg4g4KBhExEFDCRFxUI9BRBwUDCLicMAp0cYYjTNEUpi11rS3XD0GEXFQMIiIQ1yvrhw9ejTDhg07eEMRibv169ezcuXKuOwrrsFw1VVXcdttt8VzlyLSSTNnzoxbMGgoISIOCgYRcVAwiIiDbu3Ww1hrmT59ertj0R//+Mf86Ec/cqEqSTQKhh6koaGBHTt28MEHH/DFF18wYMAAAMLhMFu3bqW4uJgTTjiBgQMH4vPpn0ZPpqFED7JkyRLOPPNMPv30U4YPH86iRYv4y1/+whtvvEFmZibPPfccEyZMYPfu3W6XKi7Tn4UeJBqNEgwGATDGEAgE8Hq9pKWlARCJRGhubnazREkQ6jGIiIOCQUQcFAwi4qBgEBEHBYOIOOisRA/Sp08fLrroIpYuXUp1dTVvv/02Ho+HyspKwuEww4cPp6SkhPT0dLdLFZcpGHqQE088kdmzZ3PllVeyYMECbr755jbrL7nkEiZNmuRSdZJIFAw9jDGGhx9+mLvuusuxbtCgQd1fkCQkBUMPNGrUqIM3kh5NBx9FxEHBICIOGkokmPLycurq6hzLc3JyKCgo6PJ22dnZFBYWtlm2c+dOGhsbHW1zc3PJzc11LG9oaGDXrl0HrNsYw4ABA9q9KrP1+xUUFJCTk3PAfYn7FAwJ5te//jWvvvqqY/l1113HE0880eF2jz32GHPnznUsv+aaa3jqqadir6213H333SxatMjR9p577uEXv/iFY3lpaSk33HDDAevOyspi4cKF9OvXz7Hu3nvvZcGCBQA8+eSTXHvttQfcl7hPwZAgtm3bxrPPPsuyZcsIBALcfvvt+P1+mpubefrpp1myZAmPPvoot956K9XV1fznf/4n1113HZmZmcyaNYv+/fszefLk2P72bbd06VIeffRRbrnlFmpra5kzZw6rV68mNzeXW265BWMMNTU1/O53v+PPf/4zTU1N3HHHHfTq1Su2r2g0SlNTEwDFxcXceuutGNP2OSV+v9/RE1i/fj1//OMfWbVqVWz7SCRypH6FEkcKhgSxa9cuZsyYgbWWgQMH8tOf/pSsrCxqamqYPXs2K1euZM2aNVx66aVs3bqVGTNmMH78eAoLC3nppZd4/PHHufzyy2P7q62t5YUXXohtd8kll7B9+3ZmzJgBQElJCXfeeSder5eysjJmzZrF0qVL+frrr7npppvaBENrRUVFse06Yq2lrq6OdevW8eKLL1JfXx/fX5YccQqGJDdixAg++eSThBq3R6NRbrzxRnJycigtLeXWW29l8eLFbpclXaBgSEKDBg1i4sSJDBgwAL/fT1FRUZv1n332GaWlpdTX1zNixAjOOussCgoK8Pl8TJw4kfnz51NRUcHzzz+PMYbq6mpCoRCnnHIKp512GpmZmR2+d+vt9jnllFM45ZRTAPjmm29YvHgxo0ePZvDgwRQWFsZuBCPJQ8GQhEaMGMGUKVM67M7/+c9/Zvr06YTDYcaMGcMjjzyCz+ejuLiYKVOmsH79ehYtWsS//uu/xrYJh8Oce+653H333Y4zC8aY2LLdu3c7trvnnnsoKSnB5/OxfPlyHnjgAebPn09JSQnhcBhrv382ciQSIRQK4fP5HMcpJHFoHkMSWrJkCePHj2f58uXtrp84cSLz5s2jsLCQP/3pT5xzzjmsX7+e0tJSxo8fz7Jlyzj22GNZsGABixYtYu7cuWRkZPD8889z4YUXUl5e3mZ/Y8eOZdGiRY6vV199lYyMDGbPns2FF14Yu1dkNBrllltuYfz48YwfP56lS5fG9jVt2jQuv/xyGhoajtwvSA6begxJqK6ujrVr19LQ0EB1dTWffPIJJSUlDBw4EIDCwkICgQA+ny82v6G5uZn6+nrWrl0LQHp6OsOHD8fr9ZKdnY3H46GiooKmpibC4TDfffcdK1eu5Ic//CHNzc1s3LiRMWPGtBm25OTkOLYbOHAgF1xwQZt6KysrYwcgCwsLGTx4MB6P/iYlMv3fSSD7d61bd8H3rd+/zebNm7nxxhv56KOPsNbGvjqrvfcAWLhwITfeeCNbtmzhq6++4vrrr2fFihVt3qO99xk/fjwvv/xym68TTzwxtn7ixIn89re/JSMjo9M1SvdTjyFBDBs2jNdff53f/va3fPLJJ1x99dV4vV7C4TBVVVVMmDCB2267jcGDB1NcXMwbb7wRG9e//vrrvPvuu1x66aWx/YXDYSorKzn//PO5/fbbGTJkCP379+eNN97gkUceYd26dVx22WUYY2hubqaxsZGf/OQnXH311eTn57db4yOPPMLMmTNjr5ubm2lqauKqq67immuuOeDMTEkuCoYEkZOTw7hx41iyZAlVVVVtxuAjR47kBz/4AePGjQNaZhmeddZZsfXjxo2jtLSU2traNvvcf7t9bceMGQPQZgr1CSecwNixYznjjDOAlqnLJ510EhkZGfTq1YuTTjoJwPEeJSUlbbZrz7Bhw6ioqIjtVxKfOVC30xjT+T4p8Oijj3LbbbcddlE9WTQa7XDdgcblXdmuo7athyr7hgqtX3ekvSFOR+93sLZy6GbOnMnDDz/cpW2ste3+z1CPIcEc6kG5rmzXmbb7f4AP58OsA43JR//HRMRBwSAiDgoGEXFQMIiIg4JBRBwUDCLioGAQEQcFg4g4KBhExEHBICIOCgYRcdC1EpI0ouEwTVu2YDu4CMybmUmgf/+Ev0gr0tBAU1lZh+v9eXmk7feQoO6mYJCkEa6u5ptJk4h0cFu4XiefzLGPPdbNVXVd/bp1rL3vvg7X973iCgZOnNiNFTkpGCSp2GgU9u8xGEO/a68le+RId4rqKmudPwPgycxk4D/+I1nHH+9CUW0pGCSpeQIBfL17k3fmmWQefbTb5Rwyb3Y2acXF5J97Lr7sbLfLUTBIcss94wwG//zneAIBt0s5LP2uvpqiiy9OmJ9DZyUkKRmfj+LLLiPvrLPwZmRg4nQzmN27d/PUU0/x+eefx2V/B+Pr3Zt+111HTkkJ3vT0hDlwqh6DJB3j8+HNzqb40ktJ2+8pXIerqqqKZ555hn79+sWernWkmLQ00vr0od811+DxJdZHMbGqEemEoosvpu+VV+LPzY37vocOHcrixYs7fKhvPA26805yx47FHOABwW5RMEjS8WZkkHaE7jbt9/vp16/fEdn3/nw5Ofjz8rrlvbpKxxhExEHBICIOGkqIxEG0uZnQpk1gLcbnw3/00Ql57KCzFAwicRCtqqL6mWcgGsWTk0P+v/wLJjPT7bIOmYYSIuKgYBA5TJGKCsI7d7pdRlxpKCFymGrffJPg11+3XByVIhQMIocovHMnDQsXEt66NRYKgVNOITByJCYtzeXqDo+CQZKG8XhIKyjA6/JBPWsttq6O8LZtNC1Z0madf+hQ0seMOeD2Hr8ff0EBngQODwWDJA1f794cP2MGxu3rCqyl+oUXCG3ZckibZx13HKOfe07BIBIPxuNxvbcQ3raN5hUr8A8bhjc3l6bly7u8D+P1uv5zHIzOSoh0grUWGwwS2rqVhgULCIwaRfoPfvB9A2MgLS2pJzW1ph6DSGeEw+z53e/w9u1L/v3348nJIbRxY2y1t6CA3J/+FJMAd1+KBwWDSGcYg3/oULxFRXhycwmuWkVow4bv13u9ePLy1GMQ6UmMz0f2j38MtNyQtv7ddwlt2UKU1ByPp+LPJNItvq6o4Kp33mHZrl1ulxJ3CgaRQ5SWnU1RSQmBbrjbU3dTMIgcotElJbz22muceuqpbpcSdwqGFBUMNfH8e4/z4Vf/z+1SUlZ42zaqZ80ifIgTnRKZDj6mqIiNsPq7z/B7/W6XklKiDQ1E9+zBhkLY+nqCq1bhycvDG+e7VbtNwSDSBcFVq6iZM6fNI+ay/+EfCJSUtExyShEKhhT01cZP+HLjX2hsrnO7lNQUjZJ57rnYcJjGRYtoWrKEyPbtZE6YkDLhoGMMKWjzzjV8vOodguFmguFmqusriUTDbpeV1Ky1ROvqiDY2AuAfNgz/0KEABNesoemzz4hWVxOpqSFaW4tN8nszqMeQ4pavW8jXW5Zy72X/mwEFQ90uJ3mFQlQ9/TSR8nIAambPpvVHP1JeTuWTTwLgyc4m7557MBkZLhQaHwqGFBeJhmlsrifazmPXpQu8XtLHjCG0cSPBlSuxwSCevDzSzzrL0dQEAu5fGn6Ykrt6acNaSyjcrGFDnNlIBCIRMseNoykri+DKlQB48/PJmjChbWOvNyWul1AwpJDq+gqmv3kve+rK3S4lpTQuWULDhx8CYJubY8tDW7ZQ8cQTbdpmXXghGSkw4UnBkEIiNkJFzU6C4Sa3S0kp3txc0o4+unNtU2R6tIKhh7BEsTaKMToR1VWB0aMJjB7tdhndSv9KeoCojfL8e4/z6uLfu12KJAn1GHqIbRWbyM/p43YZkiTUYxARBwWDiDgoGFLEnrpydu3ZiiW5p+JKYtAxhhTx/5f+gU9Wv0s0GnG7FEkB6jGkiGg0qlCQuFEw9CCRSIiahirCkZDbpUiCUzD0IN+UfcVDL93A+m1/dbsUSXA6xtADGAynH/+35GTkApCXnVq3IZP4UzD0AMYYxpf8iAGF38/3D0WC+Dx+TIrccUjiS8HQA0RtlN/Pewif5/sbw/YvGMLtF01RMEi7FAwpYmjf4wlFggCUlW9kW+XmNuv3vxQ7I5AFmvMgHVAwpIizRl/MWaP/HoC3Sv/gCAaRrlAwpIh5y17mr5tLAaisTb1nKUr3UjCkiN3V29i4Y5XbZUiK0DwGEXFQjyFJWWt5+9MX2Vq+EYDNu75xuSJJJQqGJLZu219Z/d1nbpchKUhDCRFxUI+hB+mXP5hxo1pOaeZk5urGsNIhBUMPkptVyNjjzou9rmuqBsAYD1mBHM2ClBgFQw+ytuwrHnrpRsfy3KxC7r9yJgF/8j5rUeJLwZBEojZK6Zr3qWvcA0BF7c4ubR+JhqlvqnEsT/MFkv7pzBJfCoYkYm2U9z+fS1nFRrdLkRSno08i4qAegxAMN/PFhsX4vWl4PF5GDxlLmi/gdlniIgWDUN9Uwx/eb3lqc8CfwSPXv0Ca7vLUo2koISIO6jEkEYPhqKJjSE/LBFqeR9kYrHe5KklFCoYkYoyHfzz//tjr37w5SddKyBGhYEgirWcmWmu56NTrGDfqIke7ZesW8MWGxd1ZmqQYBUOSMsbQJ3cgudmFjnVrt33VpX15jJf8nD4Y4yHgT8fr8carTElSCoYkZa3lxQ+nsea7zx3ruvqoul5Z+Txw1e9jU6J9Xv9BtpBUp2BIYmccfyHHDTwZgOVrF3TpZi25WYWce9JlGGNIT8skPS1TgSAxCoYkZYzhfx07Pva6omYHu6q3dXr7PrkDOP/kK/F4dMZanBQMKeKSH97C3491XjnZEY/x6jJr6ZCCIUWkp2WSTqbbZUiKUD9SRBwUDCLioGAQEQcFg4g4KBhExEHBICIOCgYRcVAwpKBVZUv47y/+D43BOrdLkSSlYEhBO2u2sLJsCeFI0O1SJEkpGETEQVOiU0hTsJ4lG/6bTbu/drsUSXIKhhTSHG6kdP3bhCLNpHnTaQo3kB7Owq9bwUsXaSiRooKRJmYv/hULVr/idimShBQMKawxWEtzuNHtMiQJKRhSRNRGiNqu3dJNpCM6xpAiFq6ey6qyJYR0ilLiQMGQImqbqqis3+F2GZIiNJToAay1WGvdLkOSiIIhxa3b+Tl//GQKVQ073S5FkoiGEimurqmK+uZqguEmt0uRJKIeg4g4KBhExEFDiSQXjoaoa6rSUEHiSsGQ5MprtjJ78a8IRTV/QeJHwZDkLJZQpBmLTkdK/OgYQw8RDDdpuCGdpmDoAayNMnfpVN7+cpbbpUiS0FAixRVmD2Bw4UgA+vQ6yuVqJFkoGFLcUQUjuOCEm2OvI9EIHuPRk67lgBQMKW71tk8pq1rfZtlFJ97CUQUjXKpIkoGCIckFfJkc2/cUrLWEI0E2V6zC2mhsfVOojqZQ29vI6yCkHIyCIcnlZ/flqrGTAahprOB3H/yCUKTZ5aok2SkYklxF3XbeWfE81kYJR0OEIyG3S5IUoGBIcsFwI5t2rdAEJ4krzWMQEQf1GJLQjurNLN0wD2i5E7R6CxJvCoYkVNNYwVffLnS7DElhGkqIiIN6DD2CYczQ88kK5AKQl13sbjmS8BQMPYAxhpKjxtEnZ1BsWXOo5QlVPq8fr0f/DKQt/YvoAayN8krpVLwer2Pd+aNvYPTAM1yoShKZgiFJlNeW8W3FGgB21mzp8vYNwZp2l2uWpLRHwZAktpSv4u2vdD8F6R46KyEiDuox9HC7ar5j/c4vAcjP6kt+dl93C5KEoGDo4ZZueJulG94GYPxxVzL+uMtdrkgSgYYSIuKgHkOSyEjLobj3YKBlDsKehl0uVySpTMGQJI7vP5bj+p0KtDzB+pWlU12uSFKZgiFJGGMwpmWCUt/eQ/j7k251tGkM1rFwzVwi0XB3lycpRsGQhLICvTmm+BTH8tqmSjzfvEakC/tK92fh96YBEPClx6lCSXYKhiS0uXwVry2b7lhurSUU6dozLP/m+J9wwlHjAPB6/HGpT5KfgiEJ5WX24bRhfwdAbdMevtjyQZe2Lxl4JnlZLVdYDsw/loA/M+41SnJTMCShgpz+nH38VQDs2LOJlWWftLll/MGcOOhvOLpPyZEqT1KAgiHJFfU6itvO+fcubZMZ6HWEqpFUoWBIcl6Pj96ZhW6XISlGMx9FxEHBICIOCgYRcVAwiIiDgkFEHBQMIuKgYBARB81jEEkS4XCYxsZGrG37rFKPx0NWVlZc30vBIJLArLVEoy3T3fcFw/6MMaSnp8faxYOCQSSBNTc3U1tbC+DoKexjraWqqoqGhoa4va+CQSQBBINBQqGQY3k4HO4wEFrrTJuuUDCIuKT1h7m5ubndYYJbFAwiLqmtrSUYbLmxTrz/4h8uBYOIS6LRaFwPGMaT5jGIuMQY43YJHVKPQcQlWVlZZGa23FavoaGB5uZDe/K4MYbevXuTkZERt9oUDCIu8fm+//j5/f5DHlZ4PB58Ph9erzdepSkYRBJBRkZGXP/iHy4Fg0gCSLTjDTr4KCIOCgYRcVAwiIiDgkFEHBQMIuKgYBARB52ulIRmre3wAiNjTMKd5ksVCgZJaMFgkC1btrR7O7MhQ4bg9/tdqiy1KRgk4YRCIerr62Pf77s0uTVjDDU1NbFpwNnZ2W2mGMvh0W9SEk5TUxNbt249YBtrLdu3b4+9Pvroozu8VkDDja5TMEhKKCsrw+NxHksvKiqiV69eLlSU3BQMkhBaDxkO5fLjjrapr6+P9ST8fj9paWmHXmQPomCQhFBXV0dZWVnc91tRUUFFRQXQ0nsoLi6O+3ukIs1jEBEH9RgkIXg8nna7+dbadm+r3lk+ny927CGeNzJJdQoGSQi9evUiJyfHsby5uZkNGzYc8n6LiorIy8sDdHaiKxQMkhA6msXo9/vp27fvIe83Kyur3bMVcmAKBkloPp+PwsJCt8vocRSlIuKgYBARBwWDiDgoGETEQcEgIg4KBhFxUDCIiIOC4QjZXredjXs2ErWJ+ZhzkQNRMBwhb61/ixmfzSAUOfR5/iJuUTAcKRYs7d/EVCTRKRiOoIiNsLNhJ3XBOrdLEekSBcMRVNVUxUN/eYiF3y50uxSRLtFFVHFW0VjBu5veZVP1JgDC0bAOQErSUTDEWXVzNe9sfEfHFySpaSghIg4Khm6wtmotH2z5gOZw1+9+LOIGBUM3+HLXl/zX6v+iMdzodikinaJgEBEHHXyME2stayvXsq5qnQ48StJTMMTR62tfZ1XFKrfLEDlsGkqIiIOCQUQcFAwi4qBg6CbBSJDZX89m0beL3C5F5KAUDN0kYiMs37GcDdWH/rg1ke6iYBARB52ujIOttVtZvHUxOxt2ul2KSFwoGOKgvLGcBVsWAJDuTacp0uRyRSKHR8EQB6MKRjH1b6YCsKV6C1M/narZj5LUFAyHIWqjlG4rpT5UH1tW3lCuUJCkp2A4DFEb5a11b1FWV3bAdgaDx7Qc5/XoeK8kAQVDN5gwdALnDDoHgCx/lsvViBycgqEb1AXr2FG/A4CizCJy03PdLUjkIBQM3eDjso/5uOxjAM4ZfA43l9zsckUiB6YBr4g4qMfQzWqba9m4ZyMAmf5M+mb1dbkiEScFQzdbtmMZy3csB+Dk4pO5+9S7Xa5IxEnB4IJ98xw030ESlYLhMBgMeel5NEdabgtf3VxNKKqnW0vyUzAcBo/xcPepd2Nty1/+qZ9OZU3lGperEjl8CoYuCkaCzN80n6aw80Kp8sZyFyoSiT8FQxeFoiHmb5xPTbDmsPcViUZoCDUALb2PgDeAMeaw9ytyuBQMLlpVsYrJCycDMKjXIO79wb0YFAziPgWDi8LRMHua9wCQF8xztxiRVhQMnWCtJWqjALH/iqQyBUMnVDVV8ZvlvyEUDRG1UepCdW6XJHJEKRg6IWzDbK3dSjAadLsUkW6hi6hExEE9Bhcdk3sMl424DIAMX4bOSEjCUDB0g9xALrmBXMfyY/KOoaSopPsLEjkIBUM3OH/I+Vx8zMVulyHSaQqGTshJy+HmE24mYiMEI0HmrplLY7ix09sv27GMnfXOh9EM6j2IC4ZeEM9SReJCwdAJGb4Mzhx4JgANoQYWfruQ2mAt0HJFZcRGDrj95urNbK7eDEDAG/j+hrA6pCAJSsHQRRm+DB764UNYLBEb4d8+/je21W3r9Pan9T+N60ddDxC7pbxIolEwdJExhoAvALTMgpwwZEKs99AZQ3sPJd2XfqTKE4kLBcNh8BgP5w05z+0yROJOfVkRcVAwiIiDgkFEHBQMIuKgYBARBwWDiDgoGETEQcEgIg4KBhFxUDCIiIOCQUQcFAwi4qBgEBEHBYOIOCgYRMRBwSAiDrpRS4qIRqNYazvd3hgT+2rNWks02vnnc3o8Hsc+JPkpGFLEtGnTmD9/fqfbH3PMMfz+97/H6/W2Wd7Y2MhNN91EeXn5QfdRVFTEH/7wBzIzM7tcryQ2BUMSqqio4K9//SsnnHAC+fn5AGRmZpKXlxdrE4lE+PTTTykoKGD48OFttl+xYkW7vYtNmzaxZs0avvzyS9LS0hgxYsQB6+jdu7d6CylKwZCEli9fzrXXXstrr73G2WefDcAdd9zBHXfcEWtTV1fH6aefznnnncd//Md/tNn+iiuuoKKiwrHfV155hWnTpgFw3XXXMX369IPWomBITQqGJBKJRHjggQf4/PPPAXj88cd57733mDJlCs888wwfffQR0PLBP++8729SG41GefDBB8nPz2fSpEmO/VZUVDBp0iS+/vpr0tPTmTZtGiUlJUSjUX75y1+yZcuWWNvBgwczZcoUnnvuOVasWMHUqVOZN28er7/+eqyNx+NhypQpVFdX8+STTzJ58mROOukkAGbNmsWXX37JtGnTyMjIOBK/JokDBUMSsdayZMkSvv32W4477jjWrFlDKBTCWsvKlSv58MMPGTZsGNXV1e1u169fP8c+t23bxtq1a3n//ffJysriuOOOY9CgQeTn58e227RpE0cddRQAgUDLrfNXrVrFokWLiEQibNiwgffff59hw4ZRX1/Pzp07mTx5MuXl5bz33nucf/759OrVi6FDh8a2C4fDR/4XJodMwZCETj/9dF566SWuvPJK9uzZE1teXFzMvHnzyMnJob6+vlP7mjp1KnPmzCESiXDnnXfyz//8z5x99tmcc845PPXUUwCMHTuWOXPmxLbxeJxnuTMzM5k7dy4LFy7k3nvvbbPuvvvuY/To0bz33nuH8NOKGxQMSWj16tXcf//9rFu3jqKiojbrfD5fux/c/ZWVlTF58mRKS0vJzs5m0qRJnHbaafh8PsLhcJtTlt988w33338/AP379+euu+5qd58+n89xlgNahjLfffcd9913H0uXLu3CTypuUTAkoV27djFv3jz27NnjCIbOqqys5IUXXiAnJ4ehQ4dy/fXXk5WVRV1dXayNMYb8/Pw27zd8+HB+/vOfd+m99p29eOGFFwDo27fvIdUs3UczH5PQmWeeSWlpKaeeeuph7+vBBx/krbfeancugsfj4cUXX6S0tJTS0lJOO+20Q3qP6dOn89JLL7Xbm5DEpB5DEvL5fOTk5ODztf3fV1tby9NPP8348eMZOXLkAfdRVFTE1VdfzZgxY8jJyWm3jbWWN998k927dwOwdevWDiczBYNBnn32WTZu3OhYl5mZyZAhQ/jZz37G+++/36nJU+IuBUOSCQQCpKWlxb5PT295QG5aWhqhUIgZM2YQCAQYOXIk6enpbdruO6MQCAQYMmQIDz74YLt/xTMyMvD7/USjUV5++WVWrlwZWzdq1CgA/H5/7HSjz+fD5/Mxa9YsoCUIPB4PXq+XjIwMvF4vxcXFPPjgg9TU1PDhhx8eod+OxIs50Px6Y0znJ98Djz76KLfddtthFyXts9ayY8cOfD4fRUVF7Nq1i2g0SnFxMZWVlbEzEb179yYnJ4ft27eTkZFBXl5eh9vtP0EpGo2yfft20tPTyc/PZ8eOHYRCodh6v99P3759qayspKmpiX79+lFbW+s4Rdq3b18ikQi7d++mqKgoFiKVlZU0NjbSr1+/Th0klc6bOXMmDz/8cJe2sda2O0NNPYYkYoxpMxehT58+se8LCgooKCho037AgAGx7zvabn8ej6fD7Vpr/V69e/emd+/e7bYbNGhQm9f7pnBLYlNki4iDgkFEHBQMIuKgYBARBwWDiDgoGETEQcEgIg4KBhFxUDCIiIOCQUQcFAwi4qBgEBEHBYOIOCgYRMRBwSAiDgoGEXFQMIiIg4JBRBwUDCLioGAQEQcFg4g4KBhExEHBICIOB3zgjIj0TOoxiIiDgkFEHBQMIuKgYBARBwWDiDgoGETE4X8ATj1fC3vUOGQAAAAASUVORK5CYII=\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "while not done:\n",
    "  step += 1\n",
    "  \n",
    "  v_f = np.arctan2(r_f - r_a, c_f - c_a) # direction from player to target\n",
    "  spd = get_speed(observe, observe_old)\n",
    "  v_a = np.arctan2(spd, c_a - c_a_old) # speed vector of the player\n",
    "  r_a_old, c_a_old = r_a, c_a\n",
    "  observe_old = observe\n",
    "  if spd == 0 and (c_a - c_a_old) == 0:\n",
    "    # no movement\n",
    "    act = np.random.choice(3, 1)[0]\n",
    "  else:\n",
    "    if v_f - v_a < -0.1:\n",
    "      act = 1\n",
    "    elif v_f - v_a > 0.1:\n",
    "      act = 2\n",
    "    else:\n",
    "      act = 0\n",
    "      \n",
    "  observe, reward, done, info = env.step(act)\n",
    "  \n",
    "  r_a, c_a = get_pos_player(observe)\n",
    "  r_f, c_f = get_pos_flags(observe)\n",
    "  observe[int(r_a), int(c_a)] = [0, 0, 0] # draw a dot to check the player's location.\n",
    "  observe[int(r_f), int(c_f)] = [0, 0, 0] # draw a dot to check the target's location.\n",
    "  if step % 3 == 0:\n",
    "    render(observe, step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}