CW2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "matplotlib.use('TkAgg')  # avoid non-GUI warning for matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "from IPython.display import display, HTML\n",
    "from itertools import product as cartesian_product\n",
    "from skimage.draw import circle, circle_perimeter\n",
    "\n",
    "from utils import *\n",
    "\n",
    "%matplotlib inline\n",
    "np.set_printoptions(linewidth=250,precision=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Environment Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MazeGenerator(object):\n",
    "    def __init__(self):\n",
    "        self.maze = None\n",
    "    \n",
    "    def init_end_states(self):\n",
    "        \"\"\"Get start and end goals\"\"\"\n",
    "        init_state = [3,1]\n",
    "        goal_states = [[11,17]]\n",
    "        return init_state, goal_states\n",
    "        \n",
    "    def get_maze(self):\n",
    "        return self.maze\n",
    "\n",
    "\n",
    "class SimpleMazeGenerator(MazeGenerator):\n",
    "    def __init__(self, maze):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.maze = maze"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Region ID's (also related to the region colours, more in utils.py - line 113)\n",
    "regions = {'water':6, 'edge':7, 'sky':8, 'mine':4, 'uncharted':5}\n",
    "\n",
    "env_size = 19\n",
    "maze_array = np.zeros([env_size,env_size]) + regions['water']\n",
    "\n",
    "#Boundary\n",
    "maze_array[:,0] = regions['edge']\n",
    "maze_array[:,-1] = regions['edge']\n",
    "maze_array[-1,:] = regions['edge']\n",
    "maze_array[0:2,:] = regions['sky']\n",
    "\n",
    "#Bombs\n",
    "maze_array[circle(11,3,2)] = regions['mine']\n",
    "maze_array[circle(6,14,2)] = regions['mine']\n",
    "maze_array[circle(15,4,2)] = regions['mine']\n",
    "maze_array[circle(5,10,2)] = regions['mine']\n",
    "maze_array[circle(13,16,2)] = regions['mine']\n",
    "maze_array[circle(4,4,2)] = regions['mine']\n",
    "\n",
    "#Unmapped territory\n",
    "maze_array[circle(12,10,5)] = regions['uncharted']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Generate the underwater maze environment\n",
    "maze = SimpleMazeGenerator(maze_array)\n",
    "submarine = MazeEnv(maze, render_trace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Brief Introduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f06771c3f98>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQgAAAD8CAYAAACLgjpEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADq1JREFUeJzt3X+sX3V9x/HnyyIzYVhBfgi0U+MaDNPZGVJnxhaYiqVh1l9zJcuGm0uZkWQmLgtuiRD3j8vCTBacWrUBjYpkUu1mBRq2BE38QWkKBS2jIxhakCp1IOBCqu/9cU/N3eX7ae+953vv9/u99/lImu85n/O553y+96avnPP9fs55p6qQpEGeN+oBSBpfBoSkJgNCUpMBIanJgJDUZEBIajIgJDUZEJKaDAhJTSeMegCDvPDUU+uMVatHPQxpyTp04GGePHw4x+s3lgFxxqrVXPtvt4x6GNKS9YE/WD+rfr0uMZKsT3J/kv1Jrhqw/VeSfKnb/p0kL+tzPEmLa94BkWQF8DHgEuA84LIk583o9h7gJ1X168BHgX+Y7/EkLb4+ZxDrgP1V9WBVPQvcCGyc0WcjcEO3/K/AG5Ic97pH0njoExDnAA9PWz/QtQ3sU1VHgCeAF/c4pqRFNDZfcybZnGRXkl1PHn581MORRL+AOAhM/y5yVdc2sE+SE4CVwMD//VW1parOr6rzX3iqJxnSOOgTEHcCa5K8PMmJwCZg+4w+24HLu+V3Av9RPsJKmhjzngdRVUeSXAncCqwAtlbVfUk+DOyqqu3AZ4DPJdkPHGYqRCRNiF4TpapqB7BjRtuHpi3/L/CHfY4haXTG5kNKSePHgJDUZEBIajIgJDUZEJKaDAhJTQaEpCYDQlKTASGpyYCQ1GRASGoyICQ1jeVTrX/0+DP8y2d3z6rvFe967QKPRpocn7xpdv9vfvT4M7Pq5xmEpCYDQlKTASGpyYCQ1GRASGoyICQ1GRCSmvrU5lyd5D+TfC/JfUn+akCfC5M8kWRP9+9Dg/YlaTz1mSh1BPhAVe1OcjJwV5KdVfW9Gf2+UVWX9jiOpBGZ9xlEVT1aVbu75Z8C3+e5tTklTbChTLVO8jLgt4DvDNj8+iR3A48Af11V9w3jmEe945Vnzan/l/c9Ouu+b3/l2XMdzoK5ed8jox6ClqHeAZHkV4EvA++vqidnbN4NvLSqnkqyAfgKsKaxn83AZoAXrDy977AkDUGvbzGSPJ+pcPh8Vd08c3tVPVlVT3XLO4DnJzlt0L6mF+898aSVfYYlaUj6fIsRpmpvfr+q/qnR5yVdP5Ks6443sLq3pPHT5xLjd4A/AfYm2dO1/S3wawBV9QmmKnq/N8kR4GfAJqt7S5OjT3XvbwI5Tp/rgOvmewxJo+VMSklNBoSkJgNCUpMBIanJgJDUZEBIahrLx97PxVzurVB/3p+yvHgGIanJgJDUZEBIajIgJDUZEJKaDAhJTQaEpCYDQlKTASGpyYCQ1DTxU62l2XCK+Px4BiGpyYCQ1NQ7IJI8lGRvV5x314DtSfLPSfYnuSfJa/seU9LiGNZnEBdV1Y8b2y5hqprWGuB1wMe7V0ljbjEuMTYCn60p3wZelGRuBTUljcQwAqKA25Lc1dXXnOkc4OFp6wcYUAU8yeYku5LsevbpJ4YwLEl9DeMS44KqOpjkDGBnkn1Vdcdcd1JVW4AtACvPWWP1LWkM9D6DqKqD3eshYBuwbkaXg8DqaeurujZJY65vde+Tkpx8dBm4GLh3RrftwJ9232b8NvBEVfkgSWkC9L3EOBPY1hXwPgH4QlXdkuQv4ZcFfHcAG4D9wDPAn/U8pqRF0isgqupB4DUD2j8xbbmA9/U5jqTR8F6MY5ikOfPSQnCqtaQmA0JSkwEhqcmAkNRkQEhqMiAkNRkQkpoMCElNBoSkJgNCUpNTrTUnTj9fXjyDkNRkQEhqMiAkNRkQkpoMCElNBoSkJgNCUtO8AyLJuV09zqP/nkzy/hl9LkzyxLQ+H+o/ZEmLZd4TparqfmAtQJIVTNW62Dag6zeq6tL5HkfS6AzrEuMNwH9X1Q+GtD9JY2BYAbEJ+GJj2+uT3J3k60l+Y0jHk7QIet+LkeRE4C3ABwds3g28tKqeSrIB+AqwprGfzcBmgBesPL3vsDQHe2/cMuohLDzvIZmXYZxBXALsrqrHZm6oqier6qlueQfw/CSnDdpJVW2pqvOr6vwTT1o5hGFJ6msYAXEZjcuLJC9JV5cvybrueI8P4ZiSFkGvS4yuYO+bgCumtU2vy/lO4L1JjgA/AzZ1pfgkTYC+tTmfBl48o216Xc7rgOv6HEPS6DiTUlKTASGpyYCQ1GRASGoyICQ1GRCSmnzs/RK0LKZOz9Fcfyev3rR5gUYyWTyDkNRkQEhqMiAkNRkQkpoMCElNBoSkJgNCUpMBIanJgJDUZEBIajIgJDUtu3sx3v7Ks0c9BABunuNj2L2/YnHN5fe9lO/b8AxCUtOsAiLJ1iSHktw7re3UJDuTPNC9ntL42cu7Pg8kuXxYA5e08GZ7BnE9sH5G21XA7VW1Bri9W/9/kpwKXA28DlgHXN0KEknjZ1YBUVV3AIdnNG8EbuiWbwDeOuBH3wzsrKrDVfUTYCfPDRpJY6rPZxBnVtWj3fIPgTMH9DkHeHja+oGuTdIEGMqHlF21rF4Vs5JsTrIrya5nn35iGMOS1FOfgHgsyVkA3euhAX0OAqunra/q2p7D4r3S+OkTENuBo99KXA58dUCfW4GLk5zSfTh5cdcmaQLM9mvOLwLfAs5NciDJe4CPAG9K8gDwxm6dJOcn+TRAVR0G/h64s/v34a5N0gSY1UzKqrqssekNA/ruAv5i2vpWYOu8RidppJbdVGtpkrzjxrndGnDx8/59qMd3qrWkJgNCUpMBIanJgJDUZEBIajIgJDUZEJKaDAhJTQaEpCYDQlKTASGpyXsxpJ4WspTCXJ/C9OYPD/f4nkFIajIgJDUZEJKaDAhJTQaEpCYDQlLTcQOiUZfzH5PsS3JPkm1JXtT42YeS7E2yJ8muYQ5c0sKbzRnE9Ty3XN5O4FVV9ZvAfwEfPMbPX1RVa6vq/PkNUdKoHDcgBtXlrKrbqupIt/ptpgriSFpihvEZxJ8DX29sK+C2JHcl2TyEY0laRL2mWif5O+AI8PlGlwuq6mCSM4CdSfZ1ZySD9rUZ2AzwgpWn9xnWMd2875EF27e01Mz7DCLJu4FLgT/uivc+R1Ud7F4PAduAda39WZtTGj/zCogk64G/Ad5SVc80+pyU5OSjy0zV5bx3UF9J42k2X3MOqst5HXAyU5cNe5J8out7dpId3Y+eCXwzyd3Ad4GvVdUtC/IuJC2I434G0ajL+ZlG30eADd3yg8Breo1O0kg5k1JSkwEhqcmAkNRkQEhqMiAkNRkQkpoMCElNPvb+GBbyceZztfeaa0Y9BC1DnkFIajIgJDUZEJKaDAhJTQaEpCYDQlKTASGpyYCQ1GRASGoyICQ1OdV6Qrx60+zLiuy9ccsCjmR5mMvv++Y59F1wN+0e6u48g5DUNN/ivdckOdg90XpPkg2Nn12f5P4k+5NcNcyBS1p48y3eC/DRrijv2qraMXNjkhXAx4BLgPOAy5Kc12ewkhbXvIr3ztI6YH9VPVhVzwI3AhvnsR9JI9LnM4grk9zTXYKcMmD7OcDD09YPdG2SJsR8A+LjwCuAtcCjwLV9B5Jkc5JdSXY9+/QTfXcnaQjmFRBV9VhV/byqfgF8isFFeQ8Cq6etr+raWvu0eK80ZuZbvPesaatvY3BR3juBNUlenuREYBOwfT7HkzQax50o1RXvvRA4LckB4GrgwiRrgQIeAq7o+p4NfLqqNlTVkSRXArcCK4CtVXXfgrwLSQtiwYr3dus7gOd8BSppMjiTUlKT92IsQXO5jwCWx70bc/2daIpnEJKaDAhJTQaEpCYDQlKTASGpyYCQ1GRASGoyICQ1GRCSmgwISU1OtT6Gm/c9MuohLAqnIavFMwhJTQaEpCYDQlKTASGpyYCQ1GRASGoyICQ1zeap1luBS4FDVfWqru1LwLldlxcB/1NVawf87EPAT4GfA0eq6vwhjVvSIpjNRKnrgeuAzx5tqKo/Orqc5FrgWKWwLqqqH893gJJGZzaPvb8jycsGbUsS4F3A7w93WJLGQd/PIH4XeKyqHmhsL+C2JHclOeZ8XmtzSuOn770YlwFfPMb2C6rqYJIzgJ1J9lXVHYM6VtUWYAvAynPW1GwH8Mmbds9lvJLmYN5nEElOAN4OfKnVp6oOdq+HgG0MLvIraUz1ucR4I7Cvqg4M2pjkpCQnH10GLmZwkV9JY+q4AdEV7/0WcG6SA0ne023axIzLiyRnJzlai/NM4JtJ7ga+C3ytqm4Z3tAlLbT5Fu+lqt49oO2XxXur6kHgNT3HJ2mEnEkpqcmAkNRkQEhqMiAkNRkQkpoMCElNqZr1rOZFk+RHwA9mNJ8GLIe7QpfD+/Q9jt5Lq+r043Uay4AYJMmu5fA8ieXwPn2Pk8NLDElNBoSkpkkKiC2jHsAiWQ7v0/c4ISbmMwhJi2+SziAkLbKJCIgk65Pcn2R/kqtGPZ6FkOShJHuT7Emya9TjGZYkW5McSnLvtLZTk+xM8kD3esoox9hX4z1ek+Rg9/fck2TDKMc4X2MfEElWAB8DLgHOAy5Lct5oR7VgLqqqtUvh67FprgfWz2i7Cri9qtYAt3frk+x6nvseAT7a/T3XVtWOAdvH3tgHBFOPqdtfVQ9W1bPAjcDGEY9Js9Q9g/TwjOaNwA3d8g3AWxd1UEPWeI9LwiQExDnAw9PWD3RtS82snwC+BJxZVY92yz9k6uljS9GVSe7pLkEm8jJqEgJiubigql7L1KXU+5L83qgHtBhq6mu0pfhV2seBVwBrgUeBa0c7nPmZhIA4CKyetr6qa1tSltkTwB9LchZA93poxOMZuqp6rKp+XlW/AD7FhP49JyEg7gTWJHl5khOZelju9hGPaaiW4RPAtwOXd8uXA18d4VgWxNEA7LyNCf179i2cs+Cq6kiSK4FbgRXA1qq6b8TDGrYzgW1TlQw5AfjCUnkCePdU9AuB05IcAK4GPgLc1D0h/QdMlW+cWI33eGGStUxdPj0EXDGyAfbgTEpJTZNwiSFpRAwISU0GhKQmA0JSkwEhqcmAkNRkQEhqMiAkNf0fcFYfXfN5/ZQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f06771ef320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# A view of the environment\n",
    "\n",
    "#reset(), resets the environment back to it's initial conditions and returns a view of it\n",
    "snapshot = submarine.reset() \n",
    "\n",
    "plt.imshow(snapshot, cmap=submarine.cmap, norm=submarine.norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Actions :  4 \n",
      "Action ID: Up, Down, Left, Right =  [0, 1, 2, 3] \n",
      "Goal states:  [[11, 17]] \n",
      "Maze size:  (19, 19) \n",
      "List of all states in the environment:  [(0, 0), (0, 1), (0, 2), (0, 3), (0, 4)] ...\n"
     ]
    }
   ],
   "source": [
    "# Some available information from the environment\n",
    "print(\"Number of Actions : \", submarine.num_actions,\n",
    "      \"\\nAction ID: Up, Down, Left, Right = \",submarine.all_actions,\n",
    "     \"\\nGoal states: \", submarine.goal_states,\n",
    "      \"\\nMaze size: \", submarine.maze_size,\n",
    "     \"\\nList of all states in the environment: \", submarine.valid_states[0:5], \"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f06760e3940>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQgAAAD8CAYAAACLgjpEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADqxJREFUeJzt3X+sX3V9x/HnyyIzYVhBoELbqXENhunsDKkzYwtMxUKY9ddcybLh5lJmJJmJy4JbIsT947IwkwWnVm1AoyKZoN2sQMOWoIk/KE2hoGV0BEMLUqWuCLiQ6nt/3FNzd/v9tPfe8733+/32Ph9J8z3ncz73nM/33vSVc77fzznvVBWSNMjzRj0ASePLgJDUZEBIajIgJDUZEJKaDAhJTQaEpCYDQlKTASGp6aRRD2CQF55+ep21avWohyGdsA7se5SnDh7M8fqNZUCctWo11/3bbaMehnTC+sAfrJ9Vv16XGEnWJ3kwyd4kVw/Y/itJvtRt/06Sl/U5nqTFNe+ASLIM+BhwCXAecHmS82Z0ew/wk6r6deCjwD/M93iSFl+fM4h1wN6qeriqngNuAjbM6LMBuLFb/lfgDUmOe90jaTz0CYiVwKPT1vd1bQP7VNVh4BDw4h7HlLSIxuZrziSbkuxIsuOpg0+OejiS6BcQ+4Hp30Wu6toG9klyErAcGPi/v6o2V9X5VXX+C0/3JEMaB30C4m5gTZKXJzkZ2AhsndFnK3BFt/xO4D/KR1hJE2Pe8yCq6nCSq4DbgWXAlqp6IMmHgR1VtRX4DPC5JHuBg0yFiKQJ0WuiVFVtA7bNaPvQtOX/Bf6wzzEkjc7YfEgpafwYEJKaDAhJTQaEpCYDQlKTASGpyYCQ1GRASGoyICQ1GRCSmgwISU0GhKSmsXyq9Y+efJZ/+ezOWfW98l2vXeDRSJPjkzfP7v/Nj558dlb9PIOQ1GRASGoyICQ1GRCSmgwISU0GhKQmA0JSU5/anKuT/GeS7yV5IMlfDehzYZJDSXZ1/z40aF+SxlOfiVKHgQ9U1c4kpwL3JNleVd+b0e8bVXVZj+NIGpF5n0FU1eNVtbNb/inwfY6uzSlpgg1lqnWSlwG/BXxnwObXJ7kXeAz466p6YBjHXAxvf+U5ox7CL92y57FRD0FLUO+ASPKrwJeB91fVUzM27wReWlVPJ7kU+AqwprGfTcAmgBcsP7PvsCQNQa9vMZI8n6lw+HxV3TJze1U9VVVPd8vbgOcnOWPQvqYX7z35lOV9hiVpSPp8ixGmam9+v6r+qdHnJV0/kqzrjjewurek8dPnEuN3gD8BdifZ1bX9LfBrAFX1CaYqer83yWHgZ8BGq3tLk6NPde9vAjlOn+uB6+d7DEmj5UxKSU0GhKQmA0JSkwEhqcmAkNRkQEhqGsvH3mt8eX/K0uIZhKQmA0JSkwEhqcmAkNRkQEhqMiAkNRkQkpoMCElNBoSkJgNCUpNTrbUkOEV8fjyDkNRkQEhq6h0QSR5JsrsrzrtjwPYk+ecke5Pcl+S1fY8paXEM6zOIi6rqx41tlzBVTWsN8Drg492rpDG3GJcYG4DP1pRvAy9KcvYiHFdST8MIiALuSHJPV19zppXAo9PW9zGgCniSTUl2JNnx3DOHhjAsSX0N4xLjgqran+QsYHuSPVV111x3UlWbgc0Ay1eusfqWNAZ6n0FU1f7u9QBwK7BuRpf9wOpp66u6Nkljrm9171OSnHpkGbgYuH9Gt63An3bfZvw2cKiqHu9zXEmLo+8lxgrg1q6A90nAF6rqtiR/Cb8s4LsNuBTYCzwL/FnPY0paJL0CoqoeBl4zoP0T05YLeF+f40gaDe/FOIZJmjMvLQSnWktqMiAkNRkQkpoMCElNBoSkJgNCUpMBIanJgJDUZEBIajIgJDVN/FTrd7xybg+n+vIebyTtw+nnS4tnEJKaDAhJTQaEpCYDQlKTASGpyYCQ1GRASGqad0AkOberx3nk31NJ3j+jz4VJDk3r86H+Q5a0WOY9UaqqHgTWAiRZxlSti1sHdP1GVV023+NIGp1hXWK8AfjvqvrBkPYnaQwMKyA2Al9sbHt9knuTfD3JbwzpeJIWQe97MZKcDLwF+OCAzTuBl1bV00kuBb4CrGnsZxOwCeAFy8+c9fG9t6K/3TdtHvUQFp73kMzLMM4gLgF2VtUTMzdU1VNV9XS3vA14fpIzBu2kqjZX1flVdf7JpywfwrAk9TWMgLicxuVFkpekq8uXZF13vCeHcExJi6DXJUZXsPdNwJXT2qbX5Xwn8N4kh4GfARu7UnySJkDf2pzPAC+e0Ta9Luf1wPV9jiFpdJxJKanJgJDUZEBIajIgJDUZEJKaDAhJTRP/2HsdbUlMnZ6juf5OXr1x0wKNZLJ4BiGpyYCQ1GRASGoyICQ1GRCSmgwISU0GhKQmA0JSkwEhqcmAkNRkQEhqWnL3Yrz9leeMeggA3DLHx7B7f8Ximsvv+0S+b8MzCElNswqIJFuSHEhy/7S205NsT/JQ93pa42ev6Po8lOSKYQ1c0sKb7RnEDcD6GW1XA3dW1Rrgzm79/0lyOnAN8DpgHXBNK0gkjZ9ZBURV3QUcnNG8AbixW74ReOuAH30zsL2qDlbVT4DtHB00ksZUn88gVlTVkcKYPwRWDOizEnh02vq+rk3SBBjKh5RdtaxeFbOSbEqyI8mO5545NIxhSeqpT0A8keRsgO71wIA++4HV09ZXdW1HsXivNH76BMRW4Mi3ElcAXx3Q53bg4iSndR9OXty1SZoAs/2a84vAt4Bzk+xL8h7gI8CbkjwEvLFbJ8n5ST4NUFUHgb8H7u7+fbhrkzQBZjWTsqoub2x6w4C+O4C/mLa+Bdgyr9FJGqklN9VamiTvuGlutwZc/Lx/H+rxnWotqcmAkNRkQEhqMiAkNRkQkpoMCElNBoSkJgNCUpMBIanJgJDUZEBIavJeDKmnhSylMNenML35w8M9vmcQkpoMCElNBoSkJgNCUpMBIanJgJDUdNyAaNTl/Mcke5Lcl+TWJC9q/OwjSXYn2ZVkxzAHLmnhzeYM4gaOLpe3HXhVVf0m8F/AB4/x8xdV1dqqOn9+Q5Q0KscNiEF1Oavqjqo63K1+m6mCOJJOMMP4DOLPga83thVwR5J7kmwawrEkLaJeU62T/B1wGPh8o8sFVbU/yVnA9iR7ujOSQfvaBGwCeMHyM/sM65hu2fPYgu1bOtHM+wwiybuBy4A/7or3HqWq9nevB4BbgXWt/VmbUxo/8wqIJOuBvwHeUlXPNvqckuTUI8tM1eW8f1BfSeNpNl9zDqrLeT1wKlOXDbuSfKLre06Sbd2PrgC+meRe4LvA16rqtgV5F5IWxHE/g2jU5fxMo+9jwKXd8sPAa3qNTtJIOZNSUpMBIanJgJDUZEBIajIgJDUZEJKaDAhJTT72/hgW8nHmc7X72mtHPQQtQZ5BSGoyICQ1GRCSmgwISU0GhKQmA0JSkwEhqcmAkNRkQEhqMiAkNTnVekK8euPsy4rsvmnzAo5kaZjL7/uWOfRdcDfvHOruPIOQ1DTf4r3XJtnfPdF6V5JLGz+7PsmDSfYmuXqYA5e08OZbvBfgo11R3rVVtW3mxiTLgI8BlwDnAZcnOa/PYCUtrnkV752ldcDeqnq4qp4DbgI2zGM/kkakz2cQVyW5r7sEOW3A9pXAo9PW93VtkibEfAPi48ArgLXA48B1fQeSZFOSHUl2PPfMob67kzQE8wqIqnqiqn5eVb8APsXgorz7gdXT1ld1ba19WrxXGjPzLd579rTVtzG4KO/dwJokL09yMrAR2Dqf40kajeNOlOqK914InJFkH3ANcGGStUABjwBXdn3PAT5dVZdW1eEkVwG3A8uALVX1wIK8C0kLYsGK93br24CjvgKVNBmcSSmpyXsxTkBzuY8Alsa9G3P9nWiKZxCSmgwISU0GhKQmA0JSkwEhqcmAkNRkQEhqMiAkNRkQkpoMCElNTrU+hlv2PDbqISwKpyGrxTMISU0GhKQmA0JSkwEhqcmAkNRkQEhqMiAkNc3mqdZbgMuAA1X1qq7tS8C5XZcXAf9TVWsH/OwjwE+BnwOHq+r8IY1b0iKYzUSpG4Drgc8eaaiqPzqynOQ64FilsC6qqh/Pd4CSRmc2j72/K8nLBm1LEuBdwO8Pd1iSxkHfzyB+F3iiqh5qbC/gjiT3JDnmfF5rc0rjp++9GJcDXzzG9guqan+Ss4DtSfZU1V2DOlbVZmAzwPKVa2q2A/jkzTvnMl5JczDvM4gkJwFvB77U6lNV+7vXA8CtDC7yK2lM9bnEeCOwp6r2DdqY5JQkpx5ZBi5mcJFfSWPquAHRFe/9FnBukn1J3tNt2siMy4sk5yQ5UotzBfDNJPcC3wW+VlW3DW/okhbafIv3UlXvHtD2y+K9VfUw8Jqe45M0Qs6klNRkQEhqMiAkNRkQkpoMCElNBoSkplTNelbzoknyI+AHM5rPAJbCXaFL4X36HkfvpVV15vE6jWVADJJkx1J4nsRSeJ++x8nhJYakJgNCUtMkBcTmUQ9gkSyF9+l7nBAT8xmEpMU3SWcQkhbZRAREkvVJHkyyN8nVox7PQkjySJLdSXYl2THq8QxLki1JDiS5f1rb6Um2J3moez1tlGPsq/Eer02yv/t77kpy6SjHOF9jHxBJlgEfAy4BzgMuT3LeaEe1YC6qqrUnwtdj09wArJ/RdjVwZ1WtAe7s1ifZDRz9HgE+2v0911bVtgHbx97YBwRTj6nbW1UPV9VzwE3AhhGPSbPUPYP04IzmDcCN3fKNwFsXdVBD1niPJ4RJCIiVwKPT1vd1bSeaWT8B/ASwoqoe75Z/yNTTx05EVyW5r7sEmcjLqEkIiKXigqp6LVOXUu9L8nujHtBiqKmv0U7Er9I+DrwCWAs8Dlw32uHMzyQExH5g9bT1VV3bCWWJPQH8iSRnA3SvB0Y8nqGrqieq6udV9QvgU0zo33MSAuJuYE2Slyc5mamH5W4d8ZiGagk+AXwrcEW3fAXw1RGOZUEcCcDO25jQv2ffwjkLrqoOJ7kKuB1YBmypqgdGPKxhWwHcOlXJkJOAL5woTwDvnop+IXBGkn3ANcBHgJu7J6T/gKnyjROr8R4vTLKWqcunR4ArRzbAHpxJKalpEi4xJI2IASGpyYCQ1GRASGoyICQ1GRCSmgwISU0GhKSm/wNtlB9d6+MJTQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f06771730b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Unlike the traditional gym environment, each call to the step() function executes an action and takes a step through the dynamics of the environment.\n",
    "#The resulting environmental observation is returned. Note: This function updates the state of the executing agent.\n",
    "\n",
    "submarine.step(1)\n",
    "submarine.step(1)\n",
    "submarine.step(1)\n",
    "snapshot = submarine.step(1)\n",
    "plt.imshow(snapshot, cmap=submarine.cmap, norm=submarine.norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f06760c8668>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQgAAAD8CAYAAACLgjpEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADq1JREFUeJzt3X+sX3V9x/HnS5CZMKwgP4S2U+MaDNPZGVJnxhaYiqVh1l9zJcuGm0uZkWQmLgtuiRD3j8vCTBacWrUBjYpkUu1mBRq2BE38waUpv7SMjmBoQarUgYALqb73xz01d7ffT3t7z/fe7/d77/ORNN9zPudzz/l8701fOef7/ZzzTlUhSYM8b9QDkDS+DAhJTQaEpCYDQlKTASGpyYCQ1GRASGoyICQ1GRCSmk4c9QAGeeFpp9WZq1aPehjSknVg3yM8dfBgjtVvLAPizFWrufbfbhn1MKQl6wN/sH5O/XpdYiRZn+SBJHuTXDVg+68k+VK3/TtJXtbneJIW17wDIskJwMeAS4DzgMuSnDer23uAn1TVrwMfBf5hvseTtPj6nEGsA/ZW1UNV9RxwI7BxVp+NwA3d8r8Cb0hyzOseSeOhT0CsBB6Zsb6vaxvYp6oOAU8CL+5xTEmLaGy+5kyyOclUkqmnDj4x6uFIol9A7Admfhe5qmsb2CfJicAKYOD//qraUlXnV9X5LzzNkwxpHPQJiDuBNUlenuQkYBOwfVaf7cDl3fI7gf8oH2ElTYx5z4OoqkNJrgRuBU4AtlbV/Uk+DExV1XbgM8DnkuwFDjIdIpImRK+JUlW1A9gxq+1DM5b/F/jDPseQNDpj8yGlpPFjQEhqMiAkNRkQkpoMCElNBoSkJgNCUpMBIanJgJDUZEBIajIgJDUZEJKaxvKp1j964ln+5bO75tT3ine9doFHI02OT940t/83P3ri2Tn18wxCUpMBIanJgJDUZEBIajIgJDUZEJKaDAhJTX1qc65O8p9Jvpfk/iR/NaDPhUmeTLK7+/ehQfuSNJ76TJQ6BHygqnYlOQW4K8nOqvrerH7fqKpLexxH0ojM+wyiqh6rql3d8k+B73NkbU5JE2woU62TvAz4LeA7Aza/PsndwKPAX1fV/cM45mJ4+yvPGfUQfunmPY+OeghahnoHRJJfBb4MvL+qnpq1eRfw0qp6OskG4CvAmsZ+NgObAV6w4oy+w5I0BL2+xUjyfKbD4fNVdfPs7VX1VFU93S3vAJ6f5PRB+5pZvPekk1f0GZakIenzLUaYrr35/ar6p0afl3T9SLKuO97A6t6Sxk+fS4zfAf4EuDfJ7q7tb4FfA6iqTzBd0fu9SQ4BPwM2Wd1bmhx9qnt/E8gx+lwHXDffY0gaLWdSSmoyICQ1GRCSmgwISU0GhKQmA0JS01g+9l7jy/tTlhfPICQ1GRCSmgwISU0GhKQmA0JSkwEhqcmAkNRkQEhqMiAkNRkQkpqcaq1lwSni8+MZhKQmA0JSU++ASPJwknu74rxTA7YnyT8n2ZvkniSv7XtMSYtjWJ9BXFRVP25su4TpalprgNcBH+9eJY25xbjE2Ah8tqZ9G3hRkrMX4biSehpGQBRwW5K7uvqas60EHpmxvo8BVcCTbE4ylWTquWeeHMKwJPU1jEuMC6pqf5IzgZ1J9lTVHce7k6raAmwBWLFyjdW3pDHQ+wyiqvZ3rweAbcC6WV32A6tnrK/q2iSNub7VvU9OcsrhZeBi4L5Z3bYDf9p9m/HbwJNV9Vif40paHH0vMc4CtnUFvE8EvlBVtyT5S/hlAd8dwAZgL/As8Gc9jylpkfQKiKp6CHjNgPZPzFgu4H19jiNpNLwX4ygmac68tBCcai2pyYCQ1GRASGoyICQ1GRCSmgwISU0GhKQmA0JSkwEhqcmAkNTkVOshescr5/6grC/vmcwbWp1+vrx4BiGpyYCQ1GRASGoyICQ1GRCSmgwISU0GhKSmeQdEknO7epyH/z2V5P2z+lyY5MkZfT7Uf8iSFsu8J0pV1QPAWoAkJzBd62LbgK7fqKpL53scSaMzrEuMNwD/XVU/GNL+JI2BYQXEJuCLjW2vT3J3kq8n+Y0hHU/SIuh9L0aSk4C3AB8csHkX8NKqejrJBuArwJrGfjYDmwFesOKMvsMaiUm9v+LeG7eMeggLz3tI5mUYZxCXALuq6vHZG6rqqap6ulveATw/yemDdlJVW6rq/Ko6/6STVwxhWJL6GkZAXEbj8iLJS9LV5UuyrjveE0M4pqRF0OsSoyvY+ybgihltM+tyvhN4b5JDwM+ATV0pPkkToG9tzmeAF89qm1mX8zrguj7HkDQ6zqSU1GRASGoyICQ1GRCSmgwISU0GhKQmH3u/BC2LqdPH6Xh/J6/etHmBRjJZPIOQ1GRASGoyICQ1GRCSmgwISU0GhKQmA0JSkwEhqcmAkNRkQEhqMiAkNS27ezHe/spzRj0EAG4+zsewe3/F4jqe3/dSvm/DMwhJTXMKiCRbkxxIct+MttOS7EzyYPd6auNnL+/6PJjk8mENXNLCm+sZxPXA+lltVwG3V9Ua4PZu/f9JchpwNfA6YB1wdStIJI2fOQVEVd0BHJzVvBG4oVu+AXjrgB99M7Czqg5W1U+AnRwZNJLGVJ/PIM6qqsPFKH8InDWgz0rgkRnr+7o2SRNgKB9SdtWyelXMSrI5yVSSqeeeeXIYw5LUU5+AeDzJ2QDd64EBffYDq2esr+rajmDxXmn89AmI7cDhbyUuB746oM+twMVJTu0+nLy4a5M0Aeb6NecXgW8B5ybZl+Q9wEeANyV5EHhjt06S85N8GqCqDgJ/D9zZ/ftw1yZpAsxpJmVVXdbY9IYBfaeAv5ixvhXYOq/RSRqpZTfVWpok77jx+G4NuPh5/z7U4zvVWlKTASGpyYCQ1GRASGoyICQ1GRCSmgwISU0GhKQmA0JSkwEhqcmAkNTkvRhSTwtZSuF4n8L05g8P9/ieQUhqMiAkNRkQkpoMCElNBoSkJgNCUtMxA6JRl/Mfk+xJck+SbUle1PjZh5Pcm2R3kqlhDlzSwpvLGcT1HFkubyfwqqr6TeC/gA8e5ecvqqq1VXX+/IYoaVSOGRCD6nJW1W1Vdahb/TbTBXEkLTHD+Aziz4GvN7YVcFuSu5JsHsKxJC2iXlOtk/wdcAj4fKPLBVW1P8mZwM4ke7ozkkH72gxsBnjBijP6DOuobt7z6ILtW1pq5n0GkeTdwKXAH3fFe49QVfu71wPANmBda3/W5pTGz7wCIsl64G+At1TVs40+Jyc55fAy03U57xvUV9J4msvXnIPqcl4HnML0ZcPuJJ/o+p6TZEf3o2cB30xyN/Bd4GtVdcuCvAtJC+KYn0E06nJ+ptH3UWBDt/wQ8Jpeo5M0Us6klNRkQEhqMiAkNRkQkpoMCElNBoSkJgNCUpOPvT+KhXyc+fG695prRj0ELUOeQUhqMiAkNRkQkpoMCElNBoSkJgNCUpMBIanJgJDUZEBIajIgJDU51XpCvHrT3MuK3HvjlgUcyfJwPL/vm4+j74K7addQd+cZhKSm+RbvvSbJ/u6J1ruTbGj87PokDyTZm+SqYQ5c0sKbb/FegI92RXnXVtWO2RuTnAB8DLgEOA+4LMl5fQYraXHNq3jvHK0D9lbVQ1X1HHAjsHEe+5E0In0+g7gyyT3dJcipA7avBB6Zsb6va5M0IeYbEB8HXgGsBR4Dru07kCSbk0wlmXrumSf77k7SEMwrIKrq8ar6eVX9AvgUg4vy7gdWz1hf1bW19mnxXmnMzLd479kzVt/G4KK8dwJrkrw8yUnAJmD7fI4naTSOOVGqK957IXB6kn3A1cCFSdYCBTwMXNH1PQf4dFVtqKpDSa4EbgVOALZW1f0L8i4kLYgFK97bre8AjvgKVNJkcCalpCbvxViCjuc+Alge924c7+9E0zyDkNRkQEhqMiAkNRkQkpoMCElNBoSkJgNCUpMBIanJgJDUZEBIanKq9VHcvOfRUQ9hUTgNWS2eQUhqMiAkNRkQkpoMCElNBoSkJgNCUpMBIalpLk+13gpcChyoqld1bV8Czu26vAj4n6paO+BnHwZ+CvwcOFRV5w9p3JIWwVwmSl0PXAd89nBDVf3R4eUk1wJHK4V1UVX9eL4DlDQ6c3ns/R1JXjZoW5IA7wJ+f7jDkjQO+n4G8bvA41X1YGN7AbcluSvJUefzWptTGj9978W4DPjiUbZfUFX7k5wJ7Eyyp6ruGNSxqrYAWwBWrFxTcx3AJ2/adTzjlXQc5n0GkeRE4O3Al1p9qmp/93oA2MbgIr+SxlSfS4w3Anuqat+gjUlOTnLK4WXgYgYX+ZU0po4ZEF3x3m8B5ybZl+Q93aZNzLq8SHJOksO1OM8CvpnkbuC7wNeq6pbhDV3SQptv8V6q6t0D2n5ZvLeqHgJe03N8kkbImZSSmgwISU0GhKQmA0JSkwEhqcmAkNSUqjnPal40SX4E/GBW8+nAcrgrdDm8T9/j6L20qs44VqexDIhBkkwth+dJLIf36XucHF5iSGoyICQ1TVJAbBn1ABbJcnifvscJMTGfQUhafJN0BiFpkU1EQCRZn+SBJHuTXDXq8SyEJA8nuTfJ7iRTox7PsCTZmuRAkvtmtJ2WZGeSB7vXU0c5xr4a7/GaJPu7v+fuJBtGOcb5GvuASHIC8DHgEuA84LIk5412VAvmoqpauxS+HpvhemD9rLargNurag1we7c+ya7nyPcI8NHu77m2qnYM2D72xj4gmH5M3d6qeqiqngNuBDaOeEyao+4ZpAdnNW8EbuiWbwDeuqiDGrLGe1wSJiEgVgKPzFjf17UtNXN+AvgScFZVPdYt/5Dpp48tRVcmuae7BJnIy6hJCIjl4oKqei3Tl1LvS/J7ox7QYqjpr9GW4ldpHwdeAawFHgOuHe1w5mcSAmI/sHrG+qqubUlZZk8AfzzJ2QDd64ERj2foqurxqvp5Vf0C+BQT+vechIC4E1iT5OVJTmL6YbnbRzymoVqGTwDfDlzeLV8OfHWEY1kQhwOw8zYm9O/Zt3DOgquqQ0muBG4FTgC2VtX9Ix7WsJ0FbJuuZMiJwBeWyhPAu6eiXwicnmQfcDXwEeCm7gnpP2C6fOPEarzHC5OsZfry6WHgipENsAdnUkpqmoRLDEkjYkBIajIgJDUZEJKaDAhJTQaEpCYDQlKTASGp6f8AgAcfXcdDFLMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f06760ff4a8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "submarine.step(3)\n",
    "submarine.step(3)\n",
    "snapshot = submarine.step(3)\n",
    "plt.imshow(snapshot, cmap=submarine.cmap, norm=submarine.norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# _next_state(current_state,action) takes a state and an action as an input and returns the resulting state.\n",
    "# Note: This function does not update the state of the executing agent.\n",
    "submarine._next_state((0,0),3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CourseWork\n",
    "\n",
    "You are to design an agent that will be in charge of the navigation of a submarine, through a\n",
    "minefield, towards a goal.\n",
    "\n",
    "Navigating through uncharted territory can be dangerous, which we will model by applying some damage to the submarine. Additionally, there is some uncertainty in the execution of actions when moving through uncharted territory. Currents are strong and there is a 20% chance that the submarine will diverge away from its path during execution. For example, given that our current state is (12,10) within the uncharted territory, and we execute a down action; there is an 80% chance it will go down to (13,10), but also a 10% chance that it will go left to (12,9), and another 10% chance it will go right to (12,11). The uncharted territory does not have any mines and the non-determinism in action execution is limited to this area.\n",
    "\n",
    "\n",
    "Hitting a mine will lead to the destruction of the submarine and boundary regions marked **edge** or **sky** cannot be traversed.  Your task is to develop a strategy for this agent (plotted as the *blue* dot) to safely navigate towards the goal (plotted as the *green* dot).\n",
    "\n",
    "You are to do this using the principles of Dynamic Programming. In particular, in this exercise,\n",
    "you will implement the methods of Policy and Value Iteration. For all of these questions, please use\n",
    "the provided visualisation functions to plot your policies and their associated trajectories. Include\n",
    "concise comments within the notebook to highlight any interesting observations to support your\n",
    "answers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Question 1.1 : Reward and Transition Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define suitable reward and transition functions for the submarine environment. This is a key task for you as the\n",
    "designer of this agent. \n",
    "\n",
    "The reward function specifies a scalar reward for every state in the environment (in accordance to the coursework description above) and the transition function determines the next *probable* state, $s'$, reached when an action $a$ is taken from state $s$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10.],\n",
       "       [-10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10.],\n",
       "       [-10.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0., -10.],\n",
       "       [-10.,   0.,   0.,  -5.,  -5.,  -5.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0., -10.],\n",
       "       [-10.,   0.,   0.,  -5.,  -5.,  -5.,   0.,   0.,   0.,  -5.,  -5.,  -5.,   0.,   0.,   0.,   0.,   0.,   0., -10.],\n",
       "       [-10.,   0.,   0.,  -5.,  -5.,  -5.,   0.,   0.,   0.,  -5.,  -5.,  -5.,   0.,  -5.,  -5.,  -5.,   0.,   0., -10.],\n",
       "       [-10.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,  -5.,  -5.,  -5.,   0.,  -5.,  -5.,  -5.,   0.,   0., -10.],\n",
       "       [-10.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,  -5.,  -5.,  -5.,   0.,   0., -10.],\n",
       "       [-10.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,  -1.,  -1.,  -1.,  -1.,  -1.,   0.,   0.,   0.,   0.,   0., -10.],\n",
       "       [-10.,   0.,   0.,   0.,   0.,   0.,   0.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,   0.,   0.,   0.,   0., -10.],\n",
       "       [-10.,   0.,  -5.,  -5.,  -5.,   0.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,   0.,   0.,   0., -10.],\n",
       "       [-10.,   0.,  -5.,  -5.,  -5.,   0.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,   0.,   0.,   1., -10.],\n",
       "       [-10.,   0.,  -5.,  -5.,  -5.,   0.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -5.,  -5.,  -5., -10.],\n",
       "       [-10.,   0.,   0.,   0.,   0.,   0.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -5.,  -5.,  -5., -10.],\n",
       "       [-10.,   0.,   0.,  -5.,  -5.,  -5.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -5.,  -5.,  -5., -10.],\n",
       "       [-10.,   0.,   0.,  -5.,  -5.,  -5.,   0.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,  -1.,   0.,   0.,   0.,   0., -10.],\n",
       "       [-10.,   0.,   0.,  -5.,  -5.,  -5.,   0.,   0.,  -1.,  -1.,  -1.,  -1.,  -1.,   0.,   0.,   0.,   0.,   0., -10.],\n",
       "       [-10.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0., -10.],\n",
       "       [-10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10., -10.]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# HINT: Example reward function\n",
    "R"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Question 1.2: Value Iteration\n",
    "\n",
    "\n",
    "Your first algorithmic task will be to apply value iteration to the solution of this navigation task, using the reward and transition functions you have devised in the previous section."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Value iteration converged at iteration #121\n"
     ]
    }
   ],
   "source": [
    "# HINT: Example Iteration Template\n",
    "V = np.zeros_like(submarine.maze)\n",
    "V_sum = 0\n",
    "discount = 0.8\n",
    "for i in range(10000):\n",
    "    for state, region in np.ndenumerate(submarine.maze):\n",
    "        \n",
    "        #Bellman Update\n",
    "        \n",
    "        \n",
    "    if (i % 10) == 0:\n",
    "        if np.abs(V.sum() - V_sum) < 1e-6:\n",
    "            print(f'Value iteration converged at iteration #{i+1:,}')\n",
    "            break\n",
    "        V_sum = V.sum()\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f067602cf28>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQgAAAD8CAYAAACLgjpEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAEPNJREFUeJzt3X2MXNV9xvHnwWvzYlblZY15cwGnFpIbFTeynEallV0S6jfZSUmNbdQ6LdXSKEiN1KoiLYIo5Y9UhSJVRiAnsUyiBBtonRixARZaiYBIwoLMWwLFWEZ449gkpva6Adle//rHXkfb9Rx2zty5OzO7349k7cy9vz33zK79+N6Zc89xRAgAajmt1R0A0L4ICABJBASAJAICQBIBASCJgACQREAASCIgACQREACSulrdgVq6u7tj1qxZre4GMGm9++67Ghoa8nh1bRkQs2bN0h133NHqbgCT1q233lpXXalLDNtLbb9he5ftW2rsP932tmL/j2xfXuZ4ACZWwwFhe5qkeyQtkzRf0jrb88eU3SjpvYj4LUl3S/rnRo8HYOKVOYNYJGlXROyOiKOStkpaPaZmtaT7i8cPS7rG9rjXPQDaQ5mAuETSO6Oe7y221ayJiOOSDkk6v8QxAUygtvmY03av7QHbA0NDQ63uDgCVC4hBSXNGPb+02FazxnaXpN+Q9MtajUXEpohYGBELu7u7S3QLQLOUCYjnJc2zfYXtGZLWStoxpmaHpA3F489K+s9gCiugYzQ8DiIijtu+WdLjkqZJ2hwRr9n+iqSBiNgh6RuSvmV7l6SDGgkRAB2i1ECpiOiT1Ddm222jHn8g6U/LHANA67TlSMocN9xwQ2Vt534im1M/Vdo+7bT6r2KrbDunVpKmTZvWFm3n1ErSnXfemVU/nrb5FANA+yEgACQREACSCAgASQQEgCQCAkASAQEgiYAAkERAAEgiIAAkERAAkjr+XoyHH364srbb6Z6GdrlfIleV9zS0y70YufdLVNn2W2+9lVU/Hs4gACQREACSCAgASQQEgCQCAkASAQEgiYAAkFRmbc45tv/L9k9sv2b7b2rULLZ9yPbO4s9ttdoC0J7KDJQ6LulvI+JF292SXrDdHxE/GVP3g4hYWeI4AFqk4TOIiNgXES8Wj4ck/VSnrs0JoIM1Zai17csl/a6kH9XY/QnbL0n6maS/i4jXmnHMk3IX6soZhvzggw/mdqduuUN/166tf82hdhk6LVU7HDqnvp2GQ+fUd3W19m6I0ke3fbakf5f0xYg4PGb3i5Iui4gjtpdL+q6keYl2eiX1SlJPT0/ZbgFoglL/1dierpFw+HZE/MfY/RFxOCKOFI/7JE23XfNfP4v3Au2nzKcY1sjamz+NiH9N1FxY1Mn2ouJ4NVf3BtB+ylxi/L6kP5P0iu2dxbZ/kPSbkhQR92lkRe/P2z4u6X1Ja1ndG+gcZVb3fkbSh77jFxEbJW1s9BgAWouRlACSCAgASQQEgCQCAkASAQEgiYAAkNTx095XOazixIkTWfV9fX11165atSqr7ZzXmdvvnHsatm7dWlnbM2bMyGp73bp1dde2070YOfdXtPpeDM4gACQREACSCAgASQQEgCQCAkASAQEgiYAAkERAAEgiIAAkERAAkjp+qHWVcqbIl6QVK1bUXZs7RLxdhpQfO3Ysq+0nn3yy7trVq1dntT08PFx3be4SBjnDp88666ysttevX1937fTp07PabjbOIAAkERAAkkoHhO09tl8pFucdqLHftv/N9i7bL9v+WNljApgYzXoPYklE/CKxb5lGVtOaJ+njku4tvgJocxNxibFa0jdjxA8lnWP7ogk4LoCSmhEQIekJ2y8U62uOdYmkd0Y936saq4Db7rU9YHtgaGioCd0CUFYzLjGujohB2xdI6rf9ekQ8ndtIRGyStEmS5s6dy+pbQBsofQYREYPF1wOStktaNKZkUNKcUc8vLbYBaHNlV/eeabv75GNJ10p6dUzZDkl/Xnya8XuSDkXEvjLHBTAxyl5izJa0vRhx2CXpOxHxmO2/ln69gG+fpOWSdkn6laS/KHlMABOkVEBExG5JV9XYft+oxyHpC2WOA6A1Ov5ejNz7JXKsWbMmqz6nL1X2u0q5U7wvW7as7tqceyukvHtIPvjgg6y2n3vuubprr7vuuqy2c+6vyF0KoNkYag0giYAAkERAAEgiIAAkERAAkggIAEkEBIAkAgJAEgEBIImAAJDU8UOtc1U5HHoqtH399ddntX3aafX/H5RTK+UtBTBr1qystnOGT3d15f0zyhk+zbT3ANoWAQEgiYAAkERAAEgiIAAkERAAkggIAEkNB4TtK4v1OE/+OWz7i2NqFts+NKrmtvJdBjBRGh4oFRFvSFogSbanaWSti+01Sn8QESsbPQ6A1mnWJcY1kt6KiLeb1B6ANtCsgFgr6YHEvk/Yfsn2923/dpOOB2AClL4Xw/YMSaskfanG7hclXRYRR2wvl/RdSfMS7fRK6pWknp6est1Kyhm/P1W89957dddWeZ9H7hTv559/ft2169evz2q7yqnpc9o+/fTTs9putmacQSyT9GJE7B+7IyIOR8SR4nGfpOm2a/7rj4hNEbEwIhZ2d3c3oVsAympGQKxT4vLC9oUu/guxvag43i+bcEwAE6DUJUaxYO+nJN00atvodTk/K+nzto9Lel/S2uAcH+gYZdfm/F9J54/ZNnpdzo2SNpY5BoDWYSQlgCQCAkASAQEgiYAAkERAAEgiIAAkdfy097nDKnKG/nbqkI2DBw9m1R8+fLju2iqHWueOoD3jjDMqqZWkCy+8sO7a3Knpc4ZPT4ah1gAmKQICQBIBASCJgACQREAASCIgACQREACSCAgASQQEgCQCAkASAQEgqePvxci1bdu2umurvO9gzZo1WW3nTE1/6NChrLaHhobqrs39meTcp9DVlffX8f3336+kVpKOHDlSd23OfRtS3v0VufeQNBtnEACS6goI25ttH7D96qht59nut/1m8fXcxPduKGretL2hWR0HUL16zyC2SFo6Ztstkp6KiHmSniqe/z+2z5N0u6SPS1ok6fZUkABoP3UFREQ8LWnsJAOrJd1fPL5f0qdrfOsfS+qPiIMR8Z6kfp0aNADaVJn3IGZHxL7i8c8lza5Rc4mkd0Y931tsA9ABmvImZbFaVqnpl2z32h6wPZDzrjqA6pQJiP22L5Kk4uuBGjWDkuaMen5pse0ULN4LtJ8yAbFD0slPJTZI+l6NmsclXWv73OLNyWuLbQA6QL0fcz4g6TlJV9rea/tGSV+V9Cnbb0r6ZPFcthfa/rokRcRBSf8k6fniz1eKbQA6QF1D1yJiXWLXNTVqByT91ajnmyVtbqh3AFqq44da505Nn1P/yCOPZLW9atWqSvohSSdOnKi7dnh4OKvtY8eO1V2bO9T6zDPPrLs2d/r4nPoZM2ZktV3l1PQ5w6dzfn5VYKg1gCQCAkASAQEgiYAAkERAAEgiIAAkERAAkggIAEkEBIAkAgJAEgEBIKnj78Wo0sqVK7Pqc++vqKrtKvvBZD6n6u/vz6rPuRcjd9r73KUDxsMZBIAkAgJAEgEBIImAAJBEQABIIiAAJI0bEIl1Of/F9uu2X7a93fY5ie/dY/sV2zttDzSz4wCqV88ZxBadulxev6SPRsTvSPpvSV/6kO9fEhELImJhY10E0CrjBkStdTkj4omIOF48/aFGFsQBMMk04z2Iv5T0/cS+kPSE7Rds9zbhWAAmUKlxmbb/UdJxSd9OlFwdEYO2L5DUb/v14oykVlu9knolqaenp+4+5A4rXrNmTVZ9jpwp4ascDp0rZ0r9nFppagzNfvbZZ7PqDx6sf+2omTNnZrXd7L/fDZ9B2P6cpJWSbojE3/aIGCy+HpC0XdKiVHuszQm0n4YCwvZSSX8vaVVE/CpRM9N298nHGlmX89VatQDaUz0fc9Zal3OjpG6NXDbstH1fUXux7b7iW2dLesb2S5J+LOnRiHisklcBoBLjvgeRWJfzG4nan0laXjzeLemqUr0D0FKMpASQREAASCIgACQREACSCAgASQQEgCQCAkDSlJv2PuceiIceeqiyfuTctyFJS5Ysqbt2eHg4q+2cn0nuvQE5bR87diyr7cOHD2fVV+Xo0aNZ9WeffXZFPWk+ziAAJBEQAJIICABJBASAJAICQBIBASCJgACQREAASCIgACQREACSptxQ6xy5U7w/+uijddeuWLEiq+1zzz03q75dtMv0/rlDxHOGQ1e5lEKuZv+8OYMAkNTo4r1ftj1YzGi90/byxPcutf2G7V22b2lmxwFUr9HFeyXp7mJR3gUR0Td2p+1pku6RtEzSfEnrbM8v01kAE6uhxXvrtEjSrojYHRFHJW2VtLqBdgC0SJn3IG62/XJxCVLrHbRLJL0z6vneYhuADtFoQNwr6SOSFkjaJ+mush2x3Wt7wPbAVFjwFegEDQVEROyPiOGIOCHpa6q9KO+gpDmjnl9abEu1yeK9QJtpdPHei0Y9/YxqL8r7vKR5tq+wPUPSWkk7GjkegNYYd6BUsXjvYkk9tvdKul3SYtsLJIWkPZJuKmovlvT1iFgeEcdt3yzpcUnTJG2OiNcqeRUAKlHZ4r3F8z5Jp3wECqAzMJISQBL3YjRR7v0VOXKmye/U+zaqxM+kMZxBAEgiIAAkERAAkggIAEkEBIAkAgJAEgEBIImAAJBEQABIIiAAJHX8UOvcab5zhiy303TmOXJeo8QwZKRxBgEgiYAAkERAAEgiIAAkERAAkggIAEkEBICkema13ixppaQDEfHRYts2SVcWJedI+p+IWFDje/dIGpI0LOl4RCxsUr8BTIB6BkptkbRR0jdPboiI608+tn2XpEMf8v1LIuIXjXYQQOvUM+3907Yvr7XPI0P21kj6o+Z2C0A7KPsexB9I2h8Rbyb2h6QnbL9gu/fDGmJtTqD9lL0XY52kBz5k/9URMWj7Akn9tl+PiKdrFUbEJkmbJGnu3Ll132Bx4sSJnP4CyNDwGYTtLkl/ImlbqiYiBouvByRtV+1FfgG0qTKXGJ+U9HpE7K210/ZM290nH0u6VrUX+QXQpsYNiGLx3uckXWl7r+0bi11rNebywvbFtk+uxTlb0jO2X5L0Y0mPRsRjzes6gKo1univIuJzNbb9evHeiNgt6aqS/QPQQoykBJBEQABIIiAAJBEQAJIICABJBASAJOdOGz8RbL8r6e0xm3skTYW7QqfC6+Q1tt5lETFrvKK2DIhabA9MhfkkpsLr5DV2Di4xACQREACSOikgNrW6AxNkKrxOXmOH6Jj3IABMvE46gwAwwToiIGwvtf2G7V22b2l1f6pge4/tV2zvtD3Q6v40i+3Ntg/YfnXUtvNs99t+s/ja0cuLJ17jl20PFr/PnbaXt7KPjWr7gLA9TdI9kpZJmi9pne35re1VZZZExILJ8PHYKFskLR2z7RZJT0XEPElPFc872Rad+hol6e7i97kgIvpq7G97bR8QGpmmbldE7I6Io5K2Slrd4j6hTsUcpAfHbF4t6f7i8f2SPj2hnWqyxGucFDohIC6R9M6o53uLbZNN3TOATwKzI2Jf8fjnGpl9bDK62fbLxSVIR15GdUJATBVXR8THNHIp9QXbf9jqDk2EGPkYbTJ+lHavpI9IWiBpn6S7WtudxnRCQAxKmjPq+aXFtkllis0Avt/2RZJUfD3Q4v40XUTsj4jhiDgh6Wvq0N9nJwTE85Lm2b7C9gyNTJa7o8V9aqopOAP4DkkbiscbJH2vhX2pxMkALHxGHfr7LLtwTuUi4rjtmyU9LmmapM0R8VqLu9VssyVtH1nJUF2SvjNZZgAvZkVfLKnH9l5Jt0v6qqQHixnS39bI8o0dK/EaF9teoJHLpz2SbmpZB0tgJCWApE64xADQIgQEgCQCAkASAQEgiYAAkERAAEgiIAAkERAAkv4PRyuy0ul3HzsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f06760babe0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(V,cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "## HINT: You will also need to create a deterministic policy using the optimal value function\n",
    "policy = np.zeros_like(V)\n",
    "\n",
    "for state, region in np.ndenumerate(submarine.maze):\n",
    "    for action in range(submarine.num_actions):\n",
    "        next_state = tuple(submarine._next_state(state,action))\n",
    "        \n",
    "    #select the best action based on the highest state-action value\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/matplotlib/figure.py:418: UserWarning: matplotlib is currently using a non-GUI backend, so cannot show the figure\n",
      "  \"matplotlib is currently using a non-GUI backend, \"\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<video width=\"600\" height=\"400\" controls autoplay loop>\n",
       "  <source type=\"video/mp4\" src=\"data:video/mp4;base64,AAAAHGZ0eXBNNFYgAAACAGlzb21pc28yYXZjMQAAAAhmcmVlAAA3Om1kYXQAAAKsBgX//6jcRem9\n",
       "5tlIt5Ys2CDZI+7veDI2NCAtIGNvcmUgMTM4IHIyMCA0MGU3MmRhIC0gSC4yNjQvTVBFRy00IEFW\n",
       "QyBjb2RlYyAtIENvcHlsZWZ0IDIwMDMtMjAxMyAtIGh0dHA6Ly93d3cudmlkZW9sYW4ub3JnL3gy\n",
       "NjQuaHRtbCAtIG9wdGlvbnM6IGNhYmFjPTEgcmVmPTMgZGVibG9jaz0xOjA6MCBhbmFseXNlPTB4\n",
       "MzoweDExMyBtZT1oZXggc3VibWU9NyBwc3k9MSBwc3lfcmQ9MS4wMDowLjAwIG1peGVkX3JlZj0x\n",
       "IG1lX3JhbmdlPTE2IGNocm9tYV9tZT0xIHRyZWxsaXM9MSA4eDhkY3Q9MSBjcW09MCBkZWFkem9u\n",
       "ZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29mZnNldD0tMiB0aHJlYWRzPTk2IGxvb2th\n",
       "aGVhZF90aHJlYWRzPTMgc2xpY2VkX3RocmVhZHM9MCBucj0wIGRlY2ltYXRlPTEgaW50ZXJsYWNl\n",
       "ZD0wIGJsdXJheV9jb21wYXQ9MCBjb25zdHJhaW5lZF9pbnRyYT0wIGJmcmFtZXM9MyBiX3B5cmFt\n",
       "aWQ9MiBiX2FkYXB0PTEgYl9iaWFzPTAgZGlyZWN0PTEgd2VpZ2h0Yj0xIG9wZW5fZ29wPTAgd2Vp\n",
       "Z2h0cD0yIGtleWludD0yNTAga2V5aW50X21pbj01IHNjZW5lY3V0PTQwIGludHJhX3JlZnJlc2g9\n",
       "MCByY19sb29rYWhlYWQ9NDAgcmM9Y3JmIG1idHJlZT0xIGNyZj0yMy4wIHFjb21wPTAuNjAgcXBt\n",
       "aW49MCBxcG1heD02OSBxcHN0ZXA9NCBpcF9yYXRpbz0xLjQwIGFxPTE6MS4wMACAAAAKPmWIhAAS\n",
       "//73rd+BTcBA7Wu6Vzi0y6uUND1R9pSmeLZIAAADAAADAABSFy7znExAMRiOAAADA04AV8JeHwRf\n",
       "4Qbt2CiykS1MFOVCceOTpeBLnIu/0wjPi5VKDQB8iXm8U2dfLBneIAq58gYH3U/zA/a9pojy+CQl\n",
       "3iTzxWnSWGCpTv4GTYuhz0dgWNgEXbD1vi0GIwEy6nkWOReLrQxAd0ziHkyfR7/hVxZeDNA0EJ5o\n",
       "9uS3STF5wAzxaFrPer7aeoLTkynH44D3qDU6Wdzir7lVQpA0n+bj1zIIAASFQuhKUyWFXjf/02tD\n",
       "2+VJgzFy6X7UKUcsvfSXsxdhaf74jhrZIFPMr4JfDpcx12f1YM/PRqkdnRi5yhKkuNlL+6uLQWgV\n",
       "9fuyP4L3fgwdzbPvEowtn3B8FZkQsw1kxvo+/+Qg5ITEYLSvXuR6MUnelWeukkIg5XLBhg3kNzJD\n",
       "1pSNcXEn/5+AVJrV4JsDLUbsyS7fEo2cf7bvkg/gPEORObo5xAwKGQ0naR25cf5huYCHzx1t6o8/\n",
       "Z6EzE7fQjU60MPDijjepqYC+O9q1+2OsuGXl+mzm2jzKvMzmtUh8XECqA+77Fajm5ypiF9EmLjNu\n",
       "qKos7r3XJfQ8tR5UX6vFqLGUuxI8NGO1M6erW26HO+yZ182LeZudzIqCkQG65zcnNxqGtp6XiFGi\n",
       "TKMLP72muSFfQKPpim/iNimhAHGPjKdJRH0O5NT41wU0GAJFpuUxO8WIz4vmVOI5UwkIeasBoPXP\n",
       "Ta6vGC0pLCmKsWeUfAdjH0X9qgJDln/X6rWf5i+RD/8m7S/cvuuPP+2mWDuxqUrh5dtcA/5FYKHR\n",
       "NO/ZJj6XXSe9bdi+SfI8hS+1Dk0qZBlVL2V6ijsPzWWxqYpNpnexHKx980rVyN1ukDxPll5Vcumm\n",
       "6fGg1D+sWwd/ebH9IOojLFDTqRhBIw2YeUJDZlIib1T/aCMEy1XtRb86sU3JEGy2P1+Dzkc/jze6\n",
       "//8dbqJNTY/6sgtoc85WnlXBAKCrPjFBtdDY0Snyaf9YVqK9Po4UAkEII9R6vNJulFPVwIV6l3YU\n",
       "V0er9k91nqiaAYCLkoSmwwtkNEIVhumVpMIp0HGLHD0HMlMu5XmjCwnqmUPG1g4M+V4u/OsmsxPk\n",
       "haGTGjfhpngm2H96Z1NZlsGuw1mgNI9h2TB0NGX8zr2rfPeYbzCTgbHKs57RFG5X2JIXUXdrhvwQ\n",
       "o+On4ZFv9dPfBF96nyxT8T8z5rm5nFL40vb+K63nQYKPt+HzG8Qew5WlQYJhnNnzShu/Ne08mZvV\n",
       "rZDgdRTR2Ou6Gp+ZppU6YneYdipzeeTrqppJ/ZxKnJtPgW1to2o4t7Pozh5hz3i95GKdFYHSMq/A\n",
       "lvgJ4tPXtB/vkm9C992w2xQ/tXrCHPSwon0+xsBFpFV6PSYmJuCharY9RftWvbUGOU61ZR99YAVr\n",
       "DAsHFSzqtxNPkYOClfsLaIN+v4Cakm7ebDPvt2sGINfAEd3ggRlV8sS8kcv3o6VkpjmM/vRqhulc\n",
       "mlD890Ifm6GMcyTQkt5/kqqmik8Xyrq/KNVNI6uzyzoVqCiulQ8xBXrdNW1IQkZ4HH1Q+BqbbhzG\n",
       "W5rmpTtG91IH4V56s+DdzKfKe3e4LqPUG37anwXa/NG+DUPBWNvvO9Qu1Y9e2WTCAOwv3q3cno8G\n",
       "Ci/J5jrSzT/2tbWg4/durAaj7vefVKJS+WwwAhs2wGVuUZjS6n/1qd+MC/cVifhKS9p0h2FOohlK\n",
       "VDsI083TG7i+DcfvS6d3r5LBvETCGF3xA9UzAlcaxBBD4gC1qLH0Ap5E0IDEefF+d2vVgbOeA4xk\n",
       "1dB+tU9YhmU0ay9t+BK42sTl/bHS0zsKa8Gomu7IGNUzPAGPKbZchLb0QZX6ZsIG84USZXAP6Tl4\n",
       "vDZQyd1YB7ScpZ1UoviOyiaUfBGlHm3B+WoWZ9H9yj8kV7LIxPPnmfJN1f4sojWU9lU4ixb2OHy0\n",
       "e97S2feXoQUc0Ex+rp/sSI3b+j/cucpu/q+d9bIsgQgkpMTpT202aJWGMtdCByi6AAbtaSargXqY\n",
       "8rrAQTF9OwfwYI12zYg1TjPL2Q1mr73Z3aSFWp+iOR3HDDxXyvcBGPnwM/E3YB5CCE8FSVyrKXrp\n",
       "uIF9pAgp5/l9CSvUyN2jipWy2j4jbzMV39Fsc+lltCGopgnMCcLVcG3jPfiajjgC48jb738MXkEk\n",
       "T64QKEAj8YK8Nm1aJNhi4Km3WM+5A1wf2eTca2jeY6dIDfo45+kWWYjsfLPwaGDePXclGSPLV2ut\n",
       "E4QgZWF8/LjxG8NIspycZMdo4oiwkRddCXQG2kSJ5pAgS9ccrhSqhbpZoUgIPNdnHUvk479vSIgv\n",
       "c9QGqreZcC/wqyWQRoJZCCx63dQ8vjnxfzEMaPBsAS3Assnv6a0t/+xESS3vCu0o1b+msX4Ijo1x\n",
       "YZsfg9IE29JlkcU5IKrIzR+aJ6Hxf8mrDisobtwz5qJDfM83XgLVtsX32qmlpSRcf7aHHF/LaC48\n",
       "ioarJr47jfYSjNnfqPaLJCgFrAwKUKUfGECXI9yWf25HAAzuigEf8dM/1tNa0Rkg9JWQgcNLTic5\n",
       "n4EAM+UGu7I2PvMRar8hc2L9asKWzW9VcSACCmHqsID+fQjdSALCFTJnu2pj5gwkia4E4z9mQZbg\n",
       "fniz69Ff6jEuMeZ6Gc4JZ2wiNdQ/0A7eDW7N2j57NWaXX4UE87XCinaQETRhltq/T6C7ZXciYdxi\n",
       "VxWF8M7VyT2MASGSAP6fBRu+RRGGhK2LV1a64CCPF+j1OYpe6yciRNNImHdMJMj3u+F65FjaC6hh\n",
       "p1L6Ro/638mzkh/srAbQ/yk+dUl0751WEGgjntbCgp3eV2SbLvjcsyrduMGTSrsW1RK8P7oDEPNC\n",
       "pEQj0vfj4WA4bEqAuDEwilRa9q7vnYQubaL4Pd7/oqS1F4wE2pMPqKxZP3Pifev5elwLHWLCYy+J\n",
       "LRs0JPjtdtX299NfgcfD5npeNmtIGLUWuqgFQK3+gf14LIK+okLqosHb4V+ldxPAgzg8V8GcY4Zn\n",
       "9D96LNqcJAjjQkzYkgKDpmkLUaY2WuCBX7SrXjVQ0ncXA6Z8RzzMX9pPnHw9OL20YAed4vJnXwU2\n",
       "lGYTKc9x2vi1U91SptY8ZFZzs9IgFGpXqPyOkf2NB88GVoOo5nn5KZ+DGjDBmfdgrgjzorm7xoRc\n",
       "qWDGwLa6GZAoZorgTJxVPSNqxlS1+3i0D+JQiEUPaMDJm7j9hskuDyJBHOtbRDoAdY0ggQkNptuh\n",
       "gqxJMu7M1OCbT/T0Zhy+UhVHXp0Bw0kbozd2y454LGRJcyB2BG/rwhRj2uZMppNZON1TWBQTy2v1\n",
       "NpbQzbQgkHJiD4+gJIiuBz3PcT7PDNAJhgBwLDdJHqLOsDmk7tWG0G2tSJUxrnC8MKGleZkjRQyG\n",
       "cOzAMPOSo+waNucSKIF7HxBeff6QjwgjKsiRHDXR7XvYreJcuGsXKAAAAwAAAwAAAwAWkQAAASBB\n",
       "miFsQS/+tSqAAJxpHUegB/AvvXy9LiDpY+t3ohMA1D7P+R2olvipL1M9m2hDHa1qKh7zymWNfcux\n",
       "a2a2ncLxbKaLL+eMt6RwZpBjpfe1kd2nrQcaSxk3ofOzg8pHQrL7TJ1tAUNMCp7grNsuMfN1+P+8\n",
       "+x1DneCkcv2ijW03ARP7AFyHVMrDpnAdBuOXwYweDX8GXtY/cIjhRBEOgtLQIqt5P6i4PACpozHu\n",
       "HLYlhUcwuW4B3BjG0/BUbJKOrcyYh/j1V3+P8u9t3QOkG2ZZDAUojh6/wrIzaSimaAbM8ay8tyXq\n",
       "VwiTKnxOwyPz6zWrvtRmLt25R/PZlmqGxy9eepn3nKoFGr1TA+ZVAjRvef7LiBCcbY+7LEdNoCgQ\n",
       "E3AAAAJxQZpDPCGTKYQS//61KoAAkPNX8IKnkALZCckHiS/etiiybQLRrCrqm0Rhshpesr893c5/\n",
       "2hjtPYxikovmvanHMpddsiDsee1YfIun2b3C6t9SfQJPC8aJdukHnnJa9psDZ5eSRXedk8lVG0OF\n",
       "kuObCkgYcI0k/KXpWY4A4TGcSBUXuk1SgTpGHrJKV/0IWcCdtG7N1OPtqGAXsqSgPVKt1GG0nsDN\n",
       "XocSQ0iHQx1vRm5j/9dCKC5ff54vahf7A48HfBbdw3ua8Rx39icVWwqzS3FdUgFnOPUg7ivadJ7s\n",
       "ouFauCv9bHgeJdNcmhnStcAQsQWsV5qGT6VA2UA8wrkTT/iAAyND7K2QhlyusKpqWnyDs6DE4v8y\n",
       "GlI0sTEs6guDP4aS/DmgNXMF/k3eEnCKZoTOvM/FAk5MDJatRDyOycSt/pxcTox78VZSiTCA3FZu\n",
       "ptS5KQ/6v2pRVCzIF1BcWseS+3KmYpf5NkI88iWDWB9ZBxjVmqI8HR25c/YS0CBLbWVnrhESri4x\n",
       "N+DdfCYCtyLMCG9yCSPLuClg/2+mB5KY+35rhR6Vvt3cNZO4nOPsKPM3ytAxubT8jdBX/rUPx6uo\n",
       "6h6MP8sH7HHbCqy6/K4VeYwT3UTKw/6i6+n2PkAF7H8wbwdQkuhlIj/yro/GX2l31h46IRGWC5w6\n",
       "LmrqH+29tvbMyUdNtT5mFtUEBdh0dfogbbkIOV7DiYkR0S0DODPopb0O6fXnXwuUBLX4/a9NIMXG\n",
       "nX+iN2aHJSBPjVQvhdwa8I/4qZMxYmn/9heT+0BqWGtlycxFBSR2/GjDmnYxqbSlxaDywMXNnBAm\n",
       "OolaIQAAAPoBnmJqQ/8AAZHxzUBO3MEWABxvkSu8zTz5LrTJDCGYgBy/MdmTSdw0v0MfUo7D0ZaY\n",
       "tXKqTV+ugXgnA0MYDmNFoWohsbAqT03cOBFGqgybcW1c5FhjSnVvQkQwvhGzdPcaYYZY4SdLre07\n",
       "yTuCoOWkJpjlC1q3jZLNIWXuL3+jJrWoKSbfHnkd3niyLAXizBNi7BYaCZyYq3mo3/+/bHTBNSdP\n",
       "nh2fUiPYq0F0xa/UeBgnOWuuRy0mTN/5sc37k9qBAaqHvV3Wa3OlCZcjEoN3zdfUcx31VXUgsE45\n",
       "m9bQNRm4SwBJQk2goxKFUgosYieqmiZQJSW0eyEnAAABq0GaZEnhDyZTAgl//rUqgACQN7iAA3UG\n",
       "fNVmHtDg2U+xZqcnxb/yEzMw7lEq/9P5Wc9SJETwb5x8TpojDrCXfcNN2DNsiRg791WtIbZVtf35\n",
       "+xwKArCGVAoGTR+90C5gKIE4TVv+zIoHkcC0WmMcaZGEItcvVGTf22rVhF/L0zXc8DFOXhX/hTC6\n",
       "06OlG38TnWnAqkWpoK7ouoo7BfT5WzFmvLcEcEw9q76OMU0vSAY0s5BB29W8sTmkjak6VXfhm1JO\n",
       "T5UAtb97kZqJSgPHgh57+b8hocE+8KpdjwUr0w7qqRwi20xbDQF2mukV4CnFG8j1bhOTLdtu29Yz\n",
       "rwBZzfXawB4vBTd1lLoeD0y2G5PujDlvsJDBqtK2/quR/tIPRTiMjN7y2ouHBBYFvnDjuJh5DuWm\n",
       "wMAThvKvcwNCMslhO2CSXRqxn6ypPVetc86wiv4sP3pWwRoURNMBL4cVoleM9OcdW5X8L5UwU+1f\n",
       "mfKzCtnwCODReqD2aRwBVu7ieMk3IMPrCANWtps6Rbn+dyErjiYWgDIeX1wZlYzpG3/wo+I4pYFq\n",
       "X51rAjEAAAITQZqFSeEPJlMCCX/+tSqAAIA0X6eC3AB/IRkAwJytbel+BHoQsYJ1qlW/9Wf8h7AH\n",
       "LtwJekAtxa949a8an75r+ZmqL7OmkRCXG7bDZeP0pv6V/VTVmRFDk3AKlg59UGkp34LQiNnl/rO8\n",
       "CyhW8F2GcWiqFW3OfaJe14COgs4Gl4dc50amQYS/e3oStAifKarmj5XJbDwZxz568+nstpwbWa9v\n",
       "YVkS4XCHxmyms0vErXGs788qEH9NJ8iWgcNGZIJVem8aE5OaIiv7EMXtR2m//vfBnNmVstfAzmBc\n",
       "Dy8qRgQoHo26c0ERVdxZ/r3CnKh1KKLjOdSxq2navf67jsMnJXuQVUkh5ytyaZxo4BG2R625aWZx\n",
       "GcoS2OgXT7J4u7n+VJM+l6qSeurnq45bR5yyiMNk3ytPHgY+c89n85ye+25YmvUAPvQPw8COulZE\n",
       "GaMgPY63LlvkaRJm7WTgYpUw8VlNi+rHFg9uoHQmHFYP4PRRQ8hOVsDhgLAFW7TGD7DTlmcfGF2D\n",
       "HAllAWkIxB4peqWV/U3U3CcLXm4yFjJmhmjTJ1SrUkHwnvgak6yu3wF1dq7LTwg1Cg5jDcdBDdfS\n",
       "bOkiDqSgJNUaDMCuE5sIQYEDbYMyxrav4LqpyHBwFVUWammm0Lp+YAG8dP5lBc9fgg9X2cksFGRA\n",
       "zWgQ8LPjY5iEHHHNqZ/Qsd/gjyGjzPiNQHdBAAAB/EGapknhDyZTAgl//rUqgAByGct8jcAH9bDz\n",
       "hHBJUede/d1Ibwlhvv7CqvTs0nb7XjqMr38M58gW0gkkOsOlux0xCi3b0PpUbEsveiMiqDw0DM9T\n",
       "GWgJVLniIuI8PA7kuUMs/S86PvLheYBbyNZQjWcNYOFJbbJ8iPY3zzRiJ3fx5FWdJF8nB8lQg9fF\n",
       "PW9ADe5WM5dhpfabdlnnLRxnWMjl9kSgErbXPFTkPmNJcvIODWY4W1wcljjYerDdBUiCzL/c4xO1\n",
       "tGMYO7+7ZuTtHFHA+gNQw9pP7m11Hsv028ND7DJqdfdMLjPJKakJu9rxLRDT7HEOfiEtYzGsnhwo\n",
       "JvRukau+C7RiASduHmGuVOIMEsrTycfNy+lwYJMKYL+LEqwboZQQNx0VfAPTSrYsDFRfv/1cg1BH\n",
       "rVW6FIA7vctPpMeEJS/u7MqF3Ntzoz7VNyQ8nz4wYyLgizpWPVbCy2VhzYTcBuw8Z5wrP/4YeA+0\n",
       "a7IGhGKiTbiZCYS4IfV8hNyLNfSlqA7ya8Nbc+JZjRvkEADSGf9cxqnZgkzpsYj6Ph2WTBg1QFNC\n",
       "JdFtdgPRoIAFWsnNxyRPtG2uWUSTmhZyaS16Ft+jA/J+nPtDbFveE1AVmvGZqbOJbg2udomPKxIL\n",
       "vRmOBeyBBRYg7jkxmbWHYKymcMZnuOHAAz8AAADvQZrHSeEPJlMCCX/+tSqAAA6nxrKUkGawR00A\n",
       "WAqu1pxlTYQwkxC2KGbm7fBJ1MZakImIYtCsA4cYkEkPl4MzMWxihwiQxppnTeqIy/JPayugsadj\n",
       "rWwsAM4+2B8zAghmRv7IAaIE5K2JBinwrmQFksGVeuxtLX1zGjoBup7Hl4IH0lebLoyydxxghrds\n",
       "Doc+Bi8laIjScXSAurXJ8RhL/78Fd9L8PI2PS8jqLJwINR4qTAaWtg57Fd9lVYag2yVvjp+Nst2E\n",
       "hhmWF5lbqJ/d1Mq7U3MG8WnzRclO+US1C06CfvjlRy0jL0pSFWfAWkEAAAFSQZroSeEPJlMCCX/+\n",
       "tSqAAJDzV/CCp5AQlxyQcYhCUWU/wTZBqZoD7K9gU/ubd7uraIwayXQPq7ZEHY89ySj2eRw4uGFF\n",
       "1WWQ7xpi/wEvGrjmVLFWOlWosNXKNw6IA1FoCmQ/gyzJcb64eTm8HLq0Bgacx1O+l4XE7Za2QPw/\n",
       "k1fonQVIqesdeUOfAuPTRu56X2QVIZIv/A4ZOGzLsOaDITsU1a9s4Sv6P8DC2p6nPSlv+x3BxJe+\n",
       "hDUHoxC26Z5mV5fwC6EJfjN0t3Db/bSCZr1U5wRdTNFD1n0iLQERsOLHbq5379lo9kZu0TUUha0Y\n",
       "1R6jlTMIq4QwUbgVuV8SXAduUC0Gs/tCEe8iHEyuhAzI/bVuBiyxSlp5SK6BdcOzeB6N20QIV1Si\n",
       "oEc/+l7J6m/0/79DM8/QJY61jUGchNlfitOpO8D9WCWRf3Cu+owAXcAAAAGLQZsJSeEPJlMCCX/+\n",
       "tSqAAJA3uIAUJu/rpxym3PlDndBCAByLIgTZX9CkNKCBujGYzaTT7V1eXWVg/THpYXd+NVe20IO5\n",
       "GmDA9uK1m8/WnE6Wt1JGehCVVw2f3S0HjWvAnA/+X/AylCRCrMzCzNH6W05yHvtfN9yD/r84W0EU\n",
       "0X54fv1yaP6ctJqstBJphMbv3NQg/KE84MrE7swSQP3JXr4YOIIK4CdRsE7+Ojzgh6h3my8m5AP7\n",
       "9TUX/O4OkuK8v1V3VIA6SJ+3aQU862w8efrm88nxSLjaPUMzR27jtNannuHo8GXwm2FGeYMGeBTx\n",
       "3DkjMUbhcmMz6iQhDeaHrQ9rs6fiGmgnaZxwnLAM2OYDuTaA0JbSvu1SIreB0XWswuoAP2EJs07W\n",
       "tjiWnN1ca5X/l/tkYwy2tPhjSLEN9eOhrU8YxDYuQOltXMHT/J84TF8ktVOueBFme1mnqpGr5E/0\n",
       "AzvkdeVqkSSs9LnJeVdcf0L4z1Mna2lgnCEK/ertVNBObbL3QLngzIAAAAG7QZsqSeEPJlMCCX/+\n",
       "tSqAAGfl9Xz4gAVrnGqdiqyC0eCXH5FtZ7/KyoCwRer0PanOxtyqKxqqbQdeNdQTw1y2yAbLrWfd\n",
       "VeWL1SSDr2IOp1B8kQgVg/LHAcv+DmAyQJHYsCZK3zGQx4Q7JRTtaTVXjeK1ilnJ/GgXh6wFvfwB\n",
       "dTkS30P/7KHIjHUpj8WmJkB1bu4HP+er83wYVyiCO5SQdFUgI1tOZoOzvr+9QA9SZKOd+7zNZaiR\n",
       "xAB9Dh3teAeCIyWTDXy2bwO9UoCBkXO4XQlG/BbjvCpSjiLs7gxwTSYDCSwimwBRAB2rf/TPoKbv\n",
       "h5ArAzB9AIfeqltglP1za1mvtdXBbFGuoCUQbDDqWmvcBw6tRlJ1cV/GCOwjhES67QaOqy4lN8uU\n",
       "9rfVDcrkbWEJhBId+qU3lBSOMgAzdh0EbL7ZvBnRg1tpl6BbbwjAeqN4hAOHCVvFwsqDcf/zOIIl\n",
       "L9hrWAXrxGqvffApCH/dgtBq9BIQnjotDL/UvT1T72r9IlkDRCu3FoYpHlCVrnUGUY+8MAdbst7Q\n",
       "9iRNdCgOei4M75l0Z3z2sSjvqaTlCd/+1NMT1vSLBvkAAAFrQZtLSeEPJlMCCX/+tSqAAGe2LNAC\n",
       "wc41ycUqs6+wiTAPyLaz3/XX1wntHEoBJK3iDcUEHHp3E1a7JbqbUVQeIi8NF55Z1Wnlhn8RV7rZ\n",
       "mRLy55p3dAkYhixEItVNNX9ADUdSN+FcW7n/1ErAqL12viHPZKu1dVgKBv+jvDMY7kvP5HQDhDjV\n",
       "Yco8haEowuqxpWfnrGYT75kS5mqFq7SqRw9MGgDFTy9X36jZoIjpncuNMQu+wecXA51wslHQZdIV\n",
       "xZYuoshlvbdSAKfHy2Z1zWmNQHyksEATJRuWts8BNljQP3NkLRGOMCGCunm2SxbGcKFZKuCaQApz\n",
       "u3QfPkxu2ht56NNdPYL40/ygNeSptYJf1ua7p9d/a+Ug9HlAEUvURzSQ/gXGz6JnCwnxmOa1zmII\n",
       "QW4wPz59k1+Pi0Pn4WDZYnemCdc91G4AAEOVtABKblYoF/lN+3MbBEYUQK9xaLPZ1IhZzOAHOSgg\n",
       "AAABcUGbbEnhDyZTAgl//rUqgABnqEcUAIxBeolbjR5a1k4/IvjPhTVeWLsfCoGCsyYUQtyjqeHZ\n",
       "z/4ivL2+SkhKD4LZFWWzmcuSzKgmE1YcNpfMfyxv19QQlqnxcXUzGWOiZgXV9O2n9P/zmhxJMc0V\n",
       "ans1vzlLnCmZhwnyI2G9ShOa4hZl9oSVU9O473WqppG7k5yP4kbc55ZphTyytpLvlWF3XmflP+bG\n",
       "Y1ftZ4s21kQJc0i8r3NUhb+d1Uvlwo9/CW7ghVTHf0Yj46cG3OSfA2C5u18v8V7lO4DoSZxM2Kix\n",
       "LjU6vS6BzMXtwicYgc1AkbmNOgPzIRQRfsENirQ17tQLecRNP5Ub4zl+jO1gcFRcfmt7oIbsKsET\n",
       "MUsRT5TKX23lX8JyFisjEqUChv74+HIVQ7yixdSWXaBaABpXVKl7Vmlq4XJHKA1zlSUfHNnqaRGi\n",
       "AABH7w6wfnagyUK3wkaNQmWZmBKFPjyu6HuHY8+fMAAAARxBm41J4Q8mUwIJf/61KoAAZ6o7iAAt\n",
       "ymihtxpH4bVkvjPhTdbCgxqjeMN/cLSWhA6AQwHvlm9LGJi/n8Itws77eAlEqxfaOJcCpFjf4xvD\n",
       "sc9BuWZ/2i2icP/TrfwnIuvZaqDIfc+EjiOUkbu0LWn7/0RL2Z3/8RU+gnCAJMOmkNqr1IfdUzfs\n",
       "P9h0ceN+2vzHGuH5qujx1JDyn7vNWcdbLZFGsITopYCw/lfBrYm5QrCMFgpZzMIrxDZB6q+I9pni\n",
       "+RwaKBKHzyUol2D+5ViRxp0S5dvzk29VdAeIkauGKk8uiplYN3tfL4J2m/OdJ75EgQBrBlAv9mKn\n",
       "sMKW9kwMSbS7nP3GAj9BUXMY2TzJmAFW4zIAkHHJnjAggQAAAdpBm65J4Q8mUwIJf/61KoAAdTcf\n",
       "/IS5DJw/73vTO65cs5CLykfCPB4MCM/XovlgnoOfYuOvA0MFlObOg/qyCkTISwhR1duPPk6U+fX6\n",
       "TAE38wjIuHtwvYW1qrXUXckPRHHatwhyLf73YntH1iDzVfpwgcZJSC7ih6VMssu2TZTNBPUF4SX/\n",
       "PTAkhJ/guI/anyEO5Bb1uqfDmxy7ZVaDAlLUsJ0Wymany75AKmCT/Ss61iJK78tNFkk6bfQbUBAF\n",
       "u8bHfQwv91icZWdZ9/t5qU2h0aiQwGc/0AI2AuensDzw8Ic35qbDzTkdqAxewbh6JHAJxfxCJb2v\n",
       "3nWD6j5oCIQ5djwJ4W/S6hDtVhH+FH75FDr3J81n8QENKA+zWqfmJTjMdMWn0LahObOdHHlTmopo\n",
       "0lwHpSA3At37KgTmEtF7qLzSXBvztA5QL54ElodOyvAy86nDaBUStyf0XJj3fI29qEGYXktn8UBa\n",
       "uDou9MylOJlvQE9kwxDqFPkxSNB2WgTJa8XNa/OAHGFr39JlyEFvVxRjGKk/ZIZny73uLLcDQrY8\n",
       "y65Na8cFX3QXO5hBk6kZQGPQK9q2efPJ2vs1j7QYEdH/0Ief9SIBJNMC1EdD95WYWwTn17ygS8EA\n",
       "AAHqQZvPSeEPJlMCCf/+tSqAAIA1fuuwXADjAwKDSCXwd8JraKmbzwge551rtz9PGCBj7Bl8/fY4\n",
       "Si+2EO3BrNOvQh2mRHqumbm3z6C9xRZhSg7FP3TcuLxpPbKtOzNNoh9DYQERzirq8eHoaV1WGukY\n",
       "Fdm9BcpFV3j4B9y+RVkrNWwlmq1P7Wtjo65pwqGOWuoifQH8wGTgtS8u1Du+t71F//5ADtNV0dQs\n",
       "B16mnoW6jOka4u6LdEFvkUABhNIXA8N5ITzUhYJw481CewCy+fXQrt0wYZFLTAFdp3kHMqTHmGt+\n",
       "n2midY14VPzuB3C+C/0G0wEZPDRtIVrS8oYEwYf0YxUfg9OPO1x+BAvNTqEEo1JifZVrpidMbGvI\n",
       "MDAsc19IZs1EINVxQlTmm1Ml9wVPAJ7IveWCMoGY+ii04mHJm5wBrwrt34hn+/Vv/pw77Ntgc39T\n",
       "guFYb/T1RjtL1Xxn5bx/8WBF5Be6z4bqb01ea7thZOquTaLDzAoUCQXP4/i8O9ruui/qYcXI+/zn\n",
       "E/yvgj1809T4yOg7B/gp7u18JkjtbvB41aT7nlUz+YJlcAwoA+p3xEk7XtQTqIHMA2xQND1/mcDM\n",
       "dDvR+KEkGDhsW6UTmchIOaT5arAJLPKE2RFsRPqoducydH05oQAAAaRBm/NJ4Q8mUwIJf/61KoAA\n",
       "gJE2guGdjP+qJn+cZAqAB2go1GcToocuJAbDSOJlv0f9JPnpf+cz67FFsfMJxAEDfzxYAriUZ34U\n",
       "ckQz72aVzAR8cWo+ea/LJtdB34Kfk639XV5gy2DLBS5FGZW3hyt6D0M5Omw7d1ZSB+MQRv1RiUjY\n",
       "T1//rRZ7XI+zfybFV69xPAaVXF2RDKtvgPI6L//PbBHriJbFf9SGAgvqsHygtJCQnT3g257mGGB+\n",
       "9FL+VOUAifJaspiwOtuobL8IsOpiAFSd5UDAyOMVDIcKU3uXjdJjJCmJqlfrPXFfY5IIvUzKuZPa\n",
       "AO+3F041xtGAA7KrCPrGEo1DX/owXRzgP1+M/oIZDuT2ndTVDvtrISCafLvSD/20ODh0nk41fCd6\n",
       "Yb1ncX31dkoXVd3QXOJicIcKBnZBXZ79Ra4/eFF7T4FFGz2rivJ6zmFZr2uJ9iVOWPPQMrwwR9Kg\n",
       "P7IqHhIBQHjhNl2xTWAEUvXfLpSGYoXhEoBbvrusWCacMhrAh529EanPLzmxuYXJuW7eHSLmarPh\n",
       "vN0AEPAAAAHgQZ4RRRE8EP8AALN8iO74uADiQxT0s1mH2ge/PUm8g1U7g94bStN8aF9JPCdO2T6D\n",
       "bShgkABnWyfZ3TsHona8tiz8JSkqYbIjWP6KEoa93YtPl0Q3SHUJkvIw1qxDGo9jpQW1O8Ku7+9K\n",
       "Si9Vue9meIlKvZq8Z9Y1x8RSpyLEY+Y166ZBaz4YQ+91DD09UxRGI0GyKg/NRotHT7Z/uvV4qmZb\n",
       "dDa+dslkcivbOAPd+tEV5mz9Sdk5ey4ziqSCOk8XgI2GFJPzhbB9//7IiRLI4QNAINkg8XgYhVwo\n",
       "v+bc4LSQaIlfHLt1Fni1VJV3FzxhicZeW4Q7vP00zIf42CYeilIBjwGimAz+5Urxc9VtG0mrZUXs\n",
       "K7TbW193t7j/8dCECTIkpGtcn+qbVdZ9lDHC2G0/2CeBrI7Zbfh2lfTq3L2nLBNncdxFctAUEHd2\n",
       "b1C4hDj85lJQ4qmljsrDfalZZrqqRdx8b05NzUGW3zC1ypsHMrC+mzkJnrjiDZkkO/AtYyDMODvA\n",
       "1/UWY+bQPkikrfEFCXZUORd9xfhzesMSM1XPVRw3wPX7jl4hmGhtvfRrBTI9/ZdzxQKrBcP2i4Qs\n",
       "pnGVVQFd5xbeyvOF/ClSNx4RpSaJwaum41IkeUOxABRQAAAAZQGeMHRD/wABkbsfQAfvWcqY/l/g\n",
       "wJumhi4z48x7xpbSwveD3gvsr7N8PV6poUAc9SGkdkT0n0engEEuXbe9aQGh1saUYADCWxu4q88U\n",
       "63xVkmBYKyK9B9RhpYCcNMA4S+taAEXBAAABuAGeMmpD/wABkfJxwATSkSGw/l/gwMV7Go+I7bf4\n",
       "7EYg0SEYI8rLV2CIluKaOaEg31iPRyRf/YZ+1UoNVb6XajfmRgPluFUngwXheDGGM5nbF6TM9Xaf\n",
       "GBCFD+HQSNdHohe9TTkxPzI+9o9IXKJm/GaeBgDmi09Ob5M7T5xFhtUIMtlemPG6UW6OOEDeMOf/\n",
       "b4r7wbZemu+TGYDZGKa8/esTvq1pVsDZux4cwqGHRLbTZB1rWPnxUxL9RE67CrqDwlUuhD+KswEW\n",
       "k1zn3xW1C2k4MkrUWiZTpAKUwrvTw1xE4brIeonyzeQ70gRKFlHy/a45VPBMWnmqcih+tFilnvbC\n",
       "g1UpXHc+GJDezuDOyvUdVRFW2Mnkhq1Tx/TGAovSymVGcWEvXGtKeYmyFQ5E0C9QS/AFUFIZqP4Y\n",
       "rO5IWa/1j9W/FsW5Up0apiZdBmeRcvGADcQDa29v2gEuF+trzZUeCi6AyVGJnv5y5oz9PmBtLFiL\n",
       "4M///XIhuB+vShV3qG6CyNYlzckHyH8nhut833tJnPQJgyfA34q0HFeUzP4AWpGHk6y2MGC3ptYE\n",
       "sbCR88DZJqGIKDegAAABq0GaNEmoQWiZTAgl//61KoAAgCTpFO+ZOdjQCulf1A6eEezlaUtOQbY0\n",
       "nDHqaUtC4IRzLFRNNEqUAB10mP4TGw/mtGrtgcvsWUMfkYpxedTVn5MtxEeff0iMXnHk/4uuRumq\n",
       "AVhTI4+5r0gFvNwle7mwkWM99L4EJv7UUyb2gcv29vSyKt3mnhjlKfXow5TuSo//VsMHdMk8R3my\n",
       "jY/OljxuZgMDAW7bwSQDDAO8jVGiG5ILZbqF1DK6L2PGxXuXqDEso0CV1NX03IqtrOav9WrNV/5H\n",
       "Jfvrf/1BELPE1J/6xb1lez7AMCdo9sKq+QeK8cHBiD5FJHhAk3CpcaDzdtxdblcKbERkkiT1b8WO\n",
       "zcWJ5l+IRSUrp/JJlutHj2zpfOugpHDFiNm3eVldyeAUfQUNlfqfzYBBYkITYhxsdAsY6FvqPQ0K\n",
       "kPTc3iVj+Ixxrl4xSX3SzCb/QZy4ubyrtIq3d4xWtYxtRU1gaUf24xbyD2zjr2d3iWq6A66XC7IT\n",
       "S774+tIWl/VtKCCnTeqyR6T4VZhCkJBs01M+utGxcCt9B2qrk40rK19g1IAAAAFsQZpVSeEKUmUw\n",
       "IJf//rUqgACQJPUQADmwwh3Zhts258oc7oIQAORWtE2AGMtR7q2Nf8f/0HZdsZT5RAGq3BDvjOvN\n",
       "f5PHqGavAl5cUI+W9ZjD/8YQCeJog5PCFuAQShY0PByjP2bQszWt2uJKeJNhPMqq8ygSURkqyctd\n",
       "NjwUnVx6mtXDwmVimO69qf+Meg73M3B8yHgMRFGtf9oEBWncyRK5zdVhDeiODClADR0N5dx5yMZ8\n",
       "/CBwzEQnw2kBg9RKj07lp9ex5glp5/7t4c63CdTEeAhq3HwqV8Q5KXyfFzQJIkBlWox+svkEGcNx\n",
       "Ua5vxxeRcWQi2bc01YHSJhq6XQDyw2FkireHQGMh4+yyssWmvFqUppxvbZpEqJMwMJEZKVLrsTAx\n",
       "kHKy5k2+r2Y6Q1L6PUkCiSZ40hxLgGuf2VALo67gFrIdrkfpUuQ2qTuyWwIUcVntkLQFEh5PuKJ0\n",
       "pp0Jkz+xxhpbxX5eYQAAAU9BmnZJ4Q6JlMCCX/61KoAAkCGMNt9xC3VflTyJoNetel6cGv9q3mOf\n",
       "+R1jmgDpNpzeVMs0IjbwWzna1sToAPjFgoc6y5DVQMGixLOCXzwk4RuRg/6WsCCdCZrREEu1fhi4\n",
       "26j/sAAHi1m5d6VJoBfSZp0tjs/3GxYf/wkf/Tz4bAl6ct5ZPngP9Dvrojyb1pIUmtRPCvVMc0Dd\n",
       "cVdFGdmMq1r2ZvMD+3cHDGOx3eUquQxKKVj5r/lDL8OqSKHECpVoCmJtD//7jgX7dMSr5MZrN58n\n",
       "Tf7akvWoRijMhHina5nvx6FciZhLbzAqkcVk7GyMHvQWEu3t12cARlqCSiWKrZkTtqglYxFYptsN\n",
       "jthLRj2qz7UNKKnXkrsUFDUeKH7bYPIlZNF8fxBVEav8lta552lYccwMAIvhcYZERZIiTzLRj6pg\n",
       "CeaZK3UqlXAUMAAAAPZBmpdJ4Q8mUwIJf/61KoAAZ+2DagBXwHbfmGTBjHBjHgE/VRiv5+g8q4tn\n",
       "2ERrxE3hFvyrQ2XBShGa414rRxlk0PQM//f+BS5SVVTWyfCMMlKKKNHjob/sr2bSeEYbBkcYsNnh\n",
       "Hg3D1Vhum9VFJssXrsnO6fd/riegnZpnnUrsWirNR5uIWHJqEgZJL5Km6Yf+xTK+h6my7SXmL4Y0\n",
       "mfuaoTeX+j9rgVXEDGfsQ3J2AQJGCk/pMUmZvXg/sPKAvcYsb+eam5GTypmpc7TxEWrO9hbKDBes\n",
       "QP+LhovFLLtIjBYvhcxWdo+P6fd96JEzpqjuuFzi4P8AAALOQZq7SeEPJlMCH//+qZYAA32z//Ql\n",
       "tuAFvXUY8znfGkpMvXS912CBWIp/QOdN3/itWZg5uUeM+ExWykO0YZKyc+TIrRHq51M+gUF8t/tC\n",
       "fgrrmVe3WrRPn1a1R/hs6jhURUcetky0LvR5vG6rJe1BHQ+SgrqXOzVZSVQCGdgdD5sZ3eNy16mu\n",
       "fomIXTh17GWSfrfTS+kyoU8McvmRa2b3P1CEsVlCGTW5grW2SnGzlw3pjfLHYyVFT3u85BcWzEcj\n",
       "0B96YYuYm/dKPMHu3OjHckd85pyF5eQAhZBV9Uj37BdbNFA/v/VxU6lkF+COKLPpHuKlsCwNa8nY\n",
       "Nu2NKPJJHhak5xduHTxp8yGNL/gGyB3OrLVCgWidLkWmv7/mFNqjjiPHVVDp0r/RhV4ZEnrC7Vbf\n",
       "567w2eF25+2HgXomKxSyjCEa4UoUW0Ay8gSYTIamKyDvqv6LkMiQLJYNMg8o7gUT/rw5ARlPKVJt\n",
       "DL0wAlrGZ6v3cqEnjmO4Hj1+qsO4FKOCYzykHgPTfiQPPNRQycRwsNxVpB90wp91YIfJJ7EA1FE1\n",
       "/J3+Fh3JfzGCfcmu9ehwOZ0FzDlEx36x/Boiw57qZE3jQe7T72qtKV57ceFmbtdbSLVrESRX3wSR\n",
       "v0Kz9noPxItkP8wfRMS7aLB3ZnWVGhB+CRH+TNkcswEcFTx/l16FmU93jUf5FJwThI4c0TJbbaeH\n",
       "EhXv7sZXNFq3mXxiiITjOvqks+8a0IY/GUEKLHre6uX7/5VYvgi27FKhRkHLvNW5iCW16ep/fXAX\n",
       "Y13JiVE8FTMAp2oe/nzYqbcBjpE4nojp7exH0LEXDs52cFqPKDktunaPR0JXmhb3/J3L5J+6AC9B\n",
       "q0F0ULE2Uuqc9DNU9LAk72cZSMsoe7m4eMx8TZOde19GWL8XxlfyBZZfb3LmEUE8GncYWRJ5wTy3\n",
       "fvNvQg3E9SC6MMgVsQAAAjJBntlFETwQ/wAAtW36PwASyj5iNXK22rUEMkMWM03YUv1nGafY7R3/\n",
       "oa3TKLudXdQcJ4U4V3OlzIx8pEx/eN58qOeLjYpN9loCujpTqCAfQp2gNebia0UeBG6lXiKjFfUy\n",
       "gx1ChCtFp76WCs/CSOQq0sP5TsmO/vqB8AznA2v4cqiq1GySnY/pN69cPrRNO5CimWaBTvXM73kO\n",
       "MfRy5+X0a9953vEtNJzZvjax6+kxxE1bPWhOWtH4mwG+U0bEMB5rubaU9Y7SkzKInBgRLmYQ+eFg\n",
       "gqeqr7crHcfDzaeBRFCugTv3pguZb9hJuEFt3JGhYwQSBaBUfE9QzVeH+zpTXlgFn22aGMdxMUeF\n",
       "aLjSOzIN+PSfSLl6XKAcWAMS2t5PtCQW7V/iaPTWR35mVZI+Gh9xGEq8qfy36TTlEAiyBECAAG2u\n",
       "Nd41EVgW+jnegf6SBzz+m7Dd+vlW1jDibN3H+18QxL+YjxmRBlWhiEfQUlKCxlRI0m+Q+4zkxo+F\n",
       "RnIPoqwhYsns52w7ygiAjZv1X+7XqON7v0bjUC8vffJ7yux7TGFANM4+xA0OIsOI8lNvE3q5gLrd\n",
       "mZfEmXT+MX15SvjxOl2tahqE6x6HOmkuu4/jipmgQQ+aVXOyZdB0Ott5oTIba5v3bJlIysr29u5U\n",
       "ilev8+p2Y3mHgFV9P9ATJfxTMySwoZozfDPYlDYG3A/9kV8hqyH3hp0VvTAFXVp5yam0/6C/TNHI\n",
       "mAJC5aTswCbgAAAA3gGe+HRD/wABkf+dJk6dzIZ20kz+L9RCE8H1RhrvXAgBMyl8Mah6L/4Kiewg\n",
       "KFldoN3OE766Jlm/w/DafTBAcGoSKo1ywT8SDGM11TTjTD2AOUCwbPMvRmFZaFNnqBMvBvexFxr+\n",
       "E1fvrKmkBf5mLNlVscqeEM5g8h7gDwvaT9hwwi7zkruMeEs9Hml7pipikWbncrCQ4F0DlUg2qTs5\n",
       "rsu3NTkjEpGhj64ZhcTFr/+Ie+vdhVZFnHBy1flXQo/HtiTiP3HS37fgcVIXwIKgHUzbfz7yFK0r\n",
       "eSdAifZCtwAAAXEBnvpqQ/8AAZHM5d4Fh7XNRVN/XKCwdzFJJ8q/mDeMmgNER6B6Xdy7f9s4wAhT\n",
       "fWXLUV/vctsEYHyUzyaplv6nq16OBRYlti8ooJ6UDxVg8zAnm5C50l9cL0J45JogMiUCuh/H8hCT\n",
       "rd8Fgsn9Mu5++fo3lJyMALGgX5dT5NG4qpROhfE2qk7MEWLkCHKm59QSPAFcjnoDi9z/uAUqBdSB\n",
       "wPPEa1afgxcgYcqcGU6LeTDOF8wkpFQZGtyK90U8IwHxrD4yclmE3/tIaAPTfjK+Qflh+4COLU3c\n",
       "1m2HYHhQnVGvIhcjxyuyCpXAaRPFd8rQ+f0v7B0OVLSyH/nWZn3ZjMX+Vl8QoPjL/qY40WzV5GXE\n",
       "/05KmEUAgqcnq8NL3YftXV4LluekNQtwJv/6BVBNEgCcteBn+QuAyQ3zVeR2t0esOOSch7iDcST1\n",
       "nb5gCb5j+gaRwrkNJtEwL0WtquuC+kS53j3UBO6dMvCr4iH3RHwAAAQHbW9vdgAAAGxtdmhkAAAA\n",
       "AAAAAAAAAAAAAAAD6AAAFeAAAQAAAQAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAA\n",
       "AAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAzF0cmFrAAAAXHRraGQA\n",
       "AAADAAAAAAAAAAAAAAABAAAAAAAAFeAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAB\n",
       "AAAAAAAAAAAAAAAAAABAAAAAAlgAAAGQAAAAAAAkZWR0cwAAABxlbHN0AAAAAAAAAAEAABXgAAAQ\n",
       "AAABAAAAAAKpbWRpYQAAACBtZGhkAAAAAAAAAAAAAAAAAAAoAAAA4ABVxAAAAAAALWhkbHIAAAAA\n",
       "AAAAAHZpZGUAAAAAAAAAAAAAAABWaWRlb0hhbmRsZXIAAAACVG1pbmYAAAAUdm1oZAAAAAEAAAAA\n",
       "AAAAAAAAACRkaW5mAAAAHGRyZWYAAAAAAAAAAQAAAAx1cmwgAAAAAQAAAhRzdGJsAAAAtHN0c2QA\n",
       "AAAAAAAAAQAAAKRhdmMxAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAlgBkABIAAAASAAAAAAAAAAB\n",
       "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGP//AAAAMmF2Y0MBZAAW/+EAGWdkABas\n",
       "2UCYM+XhAAADAAEAAAMACg8WLZYBAAZo6+PLIsAAAAAcdXVpZGtoQPJfJE/FujmlG88DI/MAAAAA\n",
       "AAAAGHN0dHMAAAAAAAAAAQAAABwAAAgAAAAAFHN0c3MAAAAAAAAAAQAAAAEAAAB4Y3R0cwAAAAAA\n",
       "AAANAAAAAgAAEAAAAAABAAAYAAAAAAEAAAgAAAAADAAAEAAAAAABAAAoAAAAAAEAABAAAAAAAQAA\n",
       "AAAAAAABAAAIAAAAAAQAABAAAAAAAQAAKAAAAAABAAAQAAAAAAEAAAAAAAAAAQAACAAAAAAcc3Rz\n",
       "YwAAAAAAAAABAAAAAQAAABwAAAABAAAAhHN0c3oAAAAAAAAAAAAAABwAAAzyAAABJAAAAnUAAAD+\n",
       "AAABrwAAAhcAAAIAAAAA8wAAAVYAAAGPAAABvwAAAW8AAAF1AAABIAAAAd4AAAHuAAABqAAAAeQA\n",
       "AABpAAABvAAAAa8AAAFwAAABUwAAAPoAAALSAAACNgAAAOIAAAF1AAAAFHN0Y28AAAAAAAAAAQAA\n",
       "ACwAAABidWR0YQAAAFptZXRhAAAAAAAAACFoZGxyAAAAAAAAAABtZGlyYXBwbAAAAAAAAAAAAAAA\n",
       "AC1pbHN0AAAAJal0b28AAAAdZGF0YQAAAAEAAAAATGF2ZjU3LjgzLjEwMA==\n",
       "\">\n",
       "  Your browser does not support the video tag.\n",
       "</video>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAEECAYAAAC1NjdyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAABc9JREFUeJzt3T9uHGUYwOFZ5EvEDWXc0HCIUKSItEiIE8Apovw5BJIpqZCLlSLRcAk3NHFJY46xVIgCB2aiDN94fs8jbTdavV5rpZ/eON93OJ/PEwDQ9dnoAQCAscQAAMSJAQCIEwMAECcGACBODABAnBgAgDgxAABxYgAA4sQAAMSJAQCIEwMAECcGACBODABA3MXoAR7y7vc/3KsMn8iLz58cRs8wx+nO955lrm9uR4+wWb++fL7oe28zAABxYgAA4sQAAMSJAQCIEwMAECcGACBODABAnBgAgDgxAABxYgAA4sQAAMSJAQCIEwMAECcGACBODABAnBgAgDgxAABxYgAA4sQAAMSJAQCIuxg9wEN++Ol29rPff/PlipPANl3fzP+OvHj5fMVJgD2wGQCAODEAAHFiAADixAAAxIkBAIgTAwAQJwYAIE4MAECcGACAODEAAHGbPI54C45Xl6NHmKZpmk7v70ePAMDO2QwAQJwYAIA4MQAAcWIAAOLEAADEiQEAiBMDABAnBgAgTgwAQJwYAIA4MQAAce4m4D+5pwFg32wGACBODABAnBgAgDgxAABxYgAA4sQAAMSJAQCIEwMAECcGACBODABAnOOI2SVHKAPMZzMAAHFiAADixAAAxIkBAIgTAwAQJwYAIE4MAECcGACAODEAAHFiAADiHEf8AY6RBaDCZgAA4sQAAMSJAQCI8zcDABv29dWT0SNs1rO3oyfYD5sBAIgTAwAQJwYAIE4MAECcGACAODEAAHH+a+GOHJ9ervPG53XedqnjtNLPBxBnMwAAcWIAAOLEAADEiQEAiBMDABAnBgAgTgwAQJwYAIA4MQAAcWIAAOIcR7xxax0x/ObN61Xed02vXr2e//BKRyif7u7XeWOAgWwGACBODABAnBgAgDgxAABxYgAA4sQAAMSJAQCIEwMAECcGACBODABAnOOIB3DE8MdZ6+dbcszx0t+d44uBx8BmAADixAAAxIkBAIgTAwAQJwYAIE4MAECcGACAODEAAHFiAADixAAAxIkBAIhL3U1wvFrnToDFzvMf3ft9A1uw5DNeco8BwGNhMwAAcWIAAOLEAADEiQEAiBMDABAnBgAgTgwAQJwYAIA4MQAAcWIAAOJSxxHD/+34dP4R2IfD/HOqn739mGkAHmYzAABxYgAA4sQAAMSJAQCIEwMAECcGACBODABAnBgAgDgxAABxYgAA4sQALHFY73WeDrNfAJ+SGACAODEAAHFiAADixAAAxIkBAIgTAwAQdzF6AIBpmqbj1eXoETbpPHqADftq+mX0CLthMwAAcWIAAOLEAADEtf5mwD++AcA/2AwAQJwYAIA4MQAAcWIAAOLEAADEiQEAiBMDABAnBgAgTgwAQJwYAIA4MfAhh428AGBlYgAA4sQAAMSJAQCIEwMAECcGACBODABAnBgAgDgxAABxYgAA4sQAAMRdjB6Af/fFt9/Nfva3n39ccZL9WvIZnxY8u6qb29ETADtiMwAAcWIAAOLEAADEiQEAiBMDABAnBgAgTgwAQJwYAIA4MQAAcWIAAOLEAADEuZtgR9xj8LclnwVAnc0AAMSJAQCIEwMAECcGACBODABAnBgAgDgxAABxYgAA4sQAAMSJAQCISx1HfLq7n//w+/Xm2ALH9QLwF5sBAIgTAwAQJwYAIE4MAECcGACAODEAAHFiAADixAAAxIkBAIgTAwAQ9+iPI76+uR09AgA8ajYDABAnBgAgTgwAQJwYAIA4MQAAcWIAAOLEAADEiQEAiBMDABAnBgAg7nA+n0fPAAAMZDMAAHFiAADixAAAxIkBAIgTAwAQJwYAIE4MAECcGACAODEAAHFiAADixAAAxIkBAIgTAwAQJwYAIE4MAECcGACAODEAAHFiAADixAAAxIkBAIgTAwAQJwYAIE4MAECcGACAODEAAHFiAADixAAAxIkBAIgTAwAQJwYAIE4MAECcGACAODEAAHF/AhQ7X07JVAfxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f067604e3c8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# You should visulise some trajectories from a few different initial points within the environment (e.g. [23,23], [23,5], [6,4],[6,23])\n",
    "submarine.init_state = [16,2] #[3,16] #[13,2] [4,13]\n",
    "submarine.reset()\n",
    "state = submarine.init_state\n",
    "indx = 0\n",
    "while state not in submarine.goal_states:\n",
    "    submarine.step(policy[tuple(state)])\n",
    "    fig = submarine.render()\n",
    "    state = submarine.state\n",
    "    \n",
    "    if indx > 1000:\n",
    "        print(\"Max Iteration Limit Hit!\")\n",
    "        break\n",
    "    indx += 1\n",
    "anime = submarine._get_video(interval=200, gif_path='value_iter.mp4').to_html5_video()\n",
    "HTML(anime)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/matplotlib/figure.py:418: UserWarning: matplotlib is currently using a non-GUI backend, so cannot show the figure\n",
      "  \"matplotlib is currently using a non-GUI backend, \"\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<video width=\"600\" height=\"400\" controls autoplay loop>\n",
       "  <source type=\"video/mp4\" src=\"data:video/mp4;base64,AAAAHGZ0eXBNNFYgAAACAGlzb21pc28yYXZjMQAAAAhmcmVlAAA5021kYXQAAAKsBgX//6jcRem9\n",
       "5tlIt5Ys2CDZI+7veDI2NCAtIGNvcmUgMTM4IHIyMCA0MGU3MmRhIC0gSC4yNjQvTVBFRy00IEFW\n",
       "QyBjb2RlYyAtIENvcHlsZWZ0IDIwMDMtMjAxMyAtIGh0dHA6Ly93d3cudmlkZW9sYW4ub3JnL3gy\n",
       "NjQuaHRtbCAtIG9wdGlvbnM6IGNhYmFjPTEgcmVmPTMgZGVibG9jaz0xOjA6MCBhbmFseXNlPTB4\n",
       "MzoweDExMyBtZT1oZXggc3VibWU9NyBwc3k9MSBwc3lfcmQ9MS4wMDowLjAwIG1peGVkX3JlZj0x\n",
       "IG1lX3JhbmdlPTE2IGNocm9tYV9tZT0xIHRyZWxsaXM9MSA4eDhkY3Q9MSBjcW09MCBkZWFkem9u\n",
       "ZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29mZnNldD0tMiB0aHJlYWRzPTk2IGxvb2th\n",
       "aGVhZF90aHJlYWRzPTMgc2xpY2VkX3RocmVhZHM9MCBucj0wIGRlY2ltYXRlPTEgaW50ZXJsYWNl\n",
       "ZD0wIGJsdXJheV9jb21wYXQ9MCBjb25zdHJhaW5lZF9pbnRyYT0wIGJmcmFtZXM9MyBiX3B5cmFt\n",
       "aWQ9MiBiX2FkYXB0PTEgYl9iaWFzPTAgZGlyZWN0PTEgd2VpZ2h0Yj0xIG9wZW5fZ29wPTAgd2Vp\n",
       "Z2h0cD0yIGtleWludD0yNTAga2V5aW50X21pbj01IHNjZW5lY3V0PTQwIGludHJhX3JlZnJlc2g9\n",
       "MCByY19sb29rYWhlYWQ9NDAgcmM9Y3JmIG1idHJlZT0xIGNyZj0yMy4wIHFjb21wPTAuNjAgcXBt\n",
       "aW49MCBxcG1heD02OSBxcHN0ZXA9NCBpcF9yYXRpbz0xLjQwIGFxPTE6MS4wMACAAAALEWWIhAAT\n",
       "//73sY+BTcgADZc6inof4RWx9JBRerHZoGTqAAADAAADAAALO87TWwa/ZtrYAAADALEAEQCABlcx\n",
       "/CDd2wUW7SJjSTpyasRDKm0+40/JtU8wRW/AiqlqefCTF+G/QSlgtU5k4Ndt2eP+5yVTp2i2/GbZ\n",
       "3UZ+sRMomzTNCs5xBxmhz403r9eWViu2QSVKEo+PSmnsP9kZugP476rzqLgbbf/jKlFgTQKXn9w/\n",
       "Wi3s5qaXuwhIp/ejJLLT1rq/Pllc8fBTPp9jXAdnndjUDTAxD1lRjpaSlti3Bg2tscUa4177v/D1\n",
       "DJEztf5+fIpWSGuG3zYZSDphIu7fEK/AZUxLZnlTEpcDSxhWYarAAFP+py85kNKb9ruPIiEXsGZ7\n",
       "d8Hhlph/uY82MFVbC5m5l0jp5xxThypr4Z8Wl3s3AiVF447t7iuZbP+HS7qzoZC2zkIFyM+rOY0i\n",
       "eL9f1deSLPXvbDEcjpzpd0xeki2C74aT///+jnr14tPbaOXCg5jCYlc+tUgQRmXEQQB7TlOWsoTr\n",
       "Ojcv/c+BftVJQqxLW9nwqoIxjWKB7R41ooV8Hv9NbmEcQgbe8DfoBq7vM9X4ZyoyjnHwMWwVUbYC\n",
       "3+At7YhOzqSwOKm+imf8eS2UDNIMvL9OHe+wcR5ZYjWscB4a1jD9eRURSXhBVSR8mpn+eFeCJqec\n",
       "30cZRPrVV6oYa5QtImb1mM+UASVORlkb2Ll1hqWd1MJ9Hq9YB+vS2Erss0M9YqpIMK47SKSC15Yb\n",
       "ITnjyvAc8J1kD0Y/b+ZHYnrVzXkb0HIWEabbScQQWwf8/F2y6PzBfsqfAPquAOsoUcOQ9GST2UmW\n",
       "sHUIyf8G9jAS3T3Hyl1RB/RQWJ25Sg9tqYjU+KPnHaParfkvuNviMfBv/GseX4s/iZX80IbUM+td\n",
       "pEYa9blqd6swKpPFyTyDnGEWpXstJkqZWscQSAbykIsUFhsHNtZKgFhBx6FDJmwi4E4QPP0bU17b\n",
       "glWxUSvyLYT7iHjZeR0HSQw16L5QbP/5YN+iDs57cQ19JIK70XRS82wZq5IcNUuwSTib2peJMl/1\n",
       "rE9/llGwHlZrxAPnEdXaVqJ930CeHJ0h1oqFAtQbFiAwQYFH4iQ8z5AY6hXNjvcTZ8lV7F1fXHha\n",
       "prlpLhTm6UaNbXfBAbsnY5w4wyp0S4QYPmoWq1oufgAlWimUtMHhD7INNum6BXGVC0pTxQ7Nx166\n",
       "uEAGtWCc4Pl4k1GeQskfqhXTc74VLXAfEKf5r+ZUzOSn0IGwgPW/UGisTAHf5h5s0txKuPC/lroG\n",
       "kbsU7xbl1+/AA590hJrdQMuLArSpwVlniCpBrxvYn9zCoO+ntBbw26F92NnoQHO5K5gZ0y0/yJlr\n",
       "2n0vMf+NAj/odwIC7gXSK2y1mT9xvYXjoJoPP4MUh5AtwvASldbJbmKO6IZTk89+oAjlbggoPK/8\n",
       "is2yDuw6AkSBgPTIR2UENjV4Hh1THxbgtX+EgLkGulcCp5zovErOUU+wNTNGlp2vk9tCceDCX0aY\n",
       "MW0TFfZPFB3ofENdDKlEJVAS+5ZZaffByuMTEiuT4jI9N49aIErgVpbQUfMIYc9FVq68ngrt4JYt\n",
       "NKr9f96z0kmhL8W2mm9SK0VeSfnU06D6Taa9p0IjzC6ys9eYvs3ADFsHN6B2ndoFImU2s4S4KMnY\n",
       "jcx5NiAOkNTmykiwafcwKWsr6wYRmg/IK+G8+rzIyH2Mk6JF+sugP7IctzWAnbCGEhs0trZKG7wQ\n",
       "EmxoYHXAWpbU6Ibu9xGyH05LDYdfGGcjUfGeK1Cysx5oUrsCUpqJGNWWhQRGix8QQJ88lvIqU/LY\n",
       "jpPSikAqCi4yRm+zuW9TG8Tl/YS2Xn5V0rhsgLV1+4ikDrFMpEm+0V9KDeaDoiQ8iXM/Njj3XSeJ\n",
       "GF2CpMwbn5YEvzzoLnBDv4latokbmywQWtLY7W/v1+CRxRc0NeAZPLaZ8OUQY1T2h8kaN5mWDZx5\n",
       "6nRIJuTr3bHfbewyEgpsObII3aFClFapC/WK71Bph+wBh8Ox+UGioD7QSJt56idJnHEyI4aclJJ5\n",
       "s0Rf684gNt0p1+HccK5arIzwGpXL3i8amklyVKKJMFGenH9bzov2LchfcVZ+8HZKPmT8hpfAtY60\n",
       "OleNSNBxwGIIsyhO74ttVCL9Q6D7yY9BMmkN3ju1Ed+1AhBCa7SiIEDf3jlqdyMMEEXdGd+0TFch\n",
       "cyP0eCqOJo0/VwIHLcsAy3/mj6yHnkqItWiGuGhZNiXqFodaMt9Mx97e/iEI6THGfMIci7PXv3cE\n",
       "Yq7CpOWokrR+vU3IChPOzu9OJzCag2bTJtf3UE44MRPgZf3fVfHymOOt/lL1jWbfowSzpCR8G/+C\n",
       "JoD/c4sJYkTAa5v3h0oK1SLdOUIAT/ZN355U8icW4L9LImPunSRYDYEsouND4zzGANKACl7zLwok\n",
       "Hanrr/ByJX6RD9kBjHiAV6K6zBAOogqIjOdjdmQmcSsA/HFkmyMfJnXm0hRTeueCQQYItuRVVYwP\n",
       "qjQhitnB9n20w993UUZoWUGm296BZpBlEJZZB2OGuoASSAg+CmMOeul3n1W2pZF6HKhBX7h8y//3\n",
       "L9NLXGSXicDEQ0NnMg7p9IYaJedwFvSo9kYsQ4YLER/bVUWHe6bb0F73dCrK6myabZyz2Dx3BjFN\n",
       "mwio/qrg5v2/mND1AkBQA49GfuNytKJNmac3R3fYTXO/eTJMGGaSi87x9HuNGzivcpJaaJwMLhLv\n",
       "J2rM8NtKZG9EJd9Kj03uV3wWGyimsfsH9vxqA4nHdRGe46ACLgi80mOaXi3AqdWUeiF7Z0XqPsAa\n",
       "f4BZDwyXJ9QGCGq92UoPAYkUaFVrKIKF/mwnZML9rsrVyIU+14TwhEwHVTq3CHck0vX7NhZRlgh1\n",
       "SlZ0lhzYn8WFFUUW707ylyQJRNfVpgCNmP0XYhkY2IaPWfl/PIXXlVP5blnKQp+/dVwYuAIJedNG\n",
       "iFCeP5sVuXX5UfvkaYN1mxxYSX/p2jjCwFioJqpjfFaF+1VQrzj5DuxooBbeZCV5YUL0ggmRmx7s\n",
       "l0dTsmNqsBwOuCSLVO0okLCpOvBLSg24Kt29az+8vwMSY7SZT8xF9g8bpoJXu2EyoN9bnG0K1U6B\n",
       "nb/pmwBtGAGG538pRlEDzyi/+hsR+hnVkZrFDOHwsvdwr99cydq2XdTXbiOUVqCf+dDvuWP8sfne\n",
       "qT5fQBAfpSrj7yXSoebtQkFWYMfjXY1AOE1rjK97SObjI4lcsQ/YLmUtErtnKePrKiEZJAmYHRAX\n",
       "Pd7aZEbhQSWAc7CWmTDoOroSDgGc6AtmI03lbAQ19nrf+ds1nviaQkPcAs0Y/lBCfpnN3B2vKo1N\n",
       "7a4Oyq37srUIw8roKRd7GrbavOQWJJhlo6lomTtd1tA2GRRGP1X9uqxdnjuSgkuqkVEZ3eMdGV8m\n",
       "S7B0gqmE0kUxb89oPMPV2fiRGMusuZ33DStUIXc3ptgIq2+cGhS+vFatPXS3ZdcMofNsvyLqe+Z/\n",
       "iaoNZ0pjRq5J9FyXNnMv5gKI0XLTdulSsWZQHGx5ZXF21ytgwx+z4lx8JuZEkfCogvmgRkdGvftX\n",
       "VbJBv2GGkXkMSlDkT/wdkIhQtF/dbBNkr8XqNXdOCbUD7XJUl+Mk1XrDXY6a0YcBFSQGjBPD7nlM\n",
       "Ho0h0jOFYyWeKIvElKIkZ2pS4IyqxNC6Mnb1a2g3FOeoT9xD3ApBJosAzxEPEt+fpzAA+7AECt21\n",
       "y7eyAeQaIiJdBgsXTMjNqdI40AkNYSjHwAAAAwAAAwAABD0AAALDQZokbEEv/rUqgADZZLNAqB7T\n",
       "6uLxCecbdD6w1qsr58nfoGhQmbG3rT7uZuJuW1CGj3sHEYnut8tJWfXMahIzbg7Dy6koTmwJPNVC\n",
       "6/jWnkOR1nJet+6ADjfvdyskpMPmsW9DUi3+skw1maT1CMZC9LDnMSWzt3bwJJF5WKJ9qV94MaKr\n",
       "3Z5NiGD+demFx/EARycv5P+ruQ1NC3EslesiJevgcq9WcJAMbAHhoUtT8bDXQvOUrg1ylTU+GC5I\n",
       "Pdu3If79ol5PZ7wXEeUrJqOgQPzn4wKFnMqDq1OcIcvPILuqkq607eIg11JtOARSqSzO+wPcsVzG\n",
       "ndNW/lAC5aAT+9mKSMd46UYDKI1rPYLbck5yqc6JoaUD1FLVHTd3HdfVxeteE2SszCar7Jdf0qyG\n",
       "ZW3hDpkTu9T20ZBzZhUf7wXrjLHG5X7I2zZ/+XtLBlGWXytkt6zA9P9v6wrmB89VEqpZ1SjvE3ox\n",
       "fIJdZgBas7lGkJFA7XC8La7zNkIxbLAuWX44ORFl10COUDZ8QCUMyosh22pfdK8uwt0xuSBwjZii\n",
       "MNDsTJhbmkPWa4kt61h0zzy6BJAxVAAGeCJP2rSgJ9GRwin/s9D+Rnx6DRamfSy7A21NeB5nJ7v9\n",
       "WsTR/e/2eMm13cw7eW7QVp+FOslBuApxVROG31W2yXOKUXA2VBxTX+s6TPDuh2yKYgJqPnpQiYLl\n",
       "aDluz7bQAW1A6bPe/CYuvpk23rGZ+d1pbJKD3Bd6mtzaOqWRZofKIqwUQLYDqxp35rt1SqtZEcey\n",
       "khtyMQkfSOZ+S4XOK+X+JxkSVD8E02997MOG4sSIi37G62dWXMqVwNASXC5fwnH/F8qViX2mEVAz\n",
       "5F8s+Q9O2ksh5/YcyMrbbFlQ9y5yUr4Rwh4q8HJ+/Cr6UnUD+1H2KNTaycaREi92VMBqz0enouoB\n",
       "TZmJGVAAAAFjQZ5CeIIfAAENQ+WNxWe1juDKcuo9Oc+xuujfvMALo2Aft4wHYvWafow41xUs20Er\n",
       "gZ4RUWjGyhf92QbCdWfz/xJ00A8/39nFGMAaKP8OGkFuzoIOEqVp/z+3jSwZaHUSZw0St4GWi2U6\n",
       "3ctIPGlHho3ZiB71zj/QEcCCTLlI4WHfET4yJNymOemrWO118Oc4R7OpIBd+LKL9GaVHiQsMKuQ8\n",
       "w1lI09XgdR5PcU9NwySJ2oGR+0ViuBzvaNDO9N964DgmR9IU6QDsktlrDGOy3LA0BeI8YhC9BUXc\n",
       "fXpbLUwEo6k2l9U2kbV6g08nFrrdcENzc8VZJ/3gO2/s6nWW4JnlIf33eZu/qbo5ea8iVTaAJvAv\n",
       "dsKzfQ+pHyMxnH0AFzTQ7RzmrfSCaHL2NvWZ9EHM34kwTXQtwTz2U+lAgl7GVEaNaNA9R+fkLJxH\n",
       "kPvuf1CWL8V5rYkB4r0kxj9EGR4hwQAAAFYBnmF0Q/8AAlq5XzJWPCMToKXaOq+Lt+gB9ACdkQVT\n",
       "isarKY6Sxz25soOdeWYvCxsew8fkLWHcOjMGm2q+jCAYtL9MdFdH/6KXJrKbSuPerNEw1tB8wAAA\n",
       "AOsBnmNqQ/8AAWe0VHN3a+f2sDB1q/6fGf9HmtwAWGTn7hjFUxwAgH/ttbr6xMncu+HqswIJccJm\n",
       "6mo5YpHfX4AI1OkjHONbp3PqB4X2R+weV7m5BiD+hlIT3yYKBCP0uH6MwV/gb78DU8S5gbIqXgRO\n",
       "Ly2gyvp2e4+/wLOImTbDsbOgJqAIogVN7Iish6onpI4ujfktW7xsmY2jFi/Rb+6Tr4gKtLrfnXUO\n",
       "VZNlZ+2c+Ya26oQjlMtrGMxEntpT/PBRhotBS9Q/HL7ldHBNaD3gGIGoypeitrZRZ7X/ePejw8aY\n",
       "Awo9ptI66AKTAAAB5EGaZUmoQWiZTAgl//61KoAAkPNX8IKnkALZCckHiS/etiiybQLRrCrqm0Rh\n",
       "shpesr893c5/2hjtPYxikovmvanHMpddsiDsee4KVApBl5D8vpCNuOKtOeFzgOUdJAoFkiqT1ko4\n",
       "uKfrnu4fXun78UPh3xzTOMxYQSe8YbSewM1ehxI8itjiHh1OK7zb/K8RQb4cluLwATjYHHg74Lbu\n",
       "G9t51/trq/hhVRl43EFzuyeqg6KpnJ1S/huV9Abw+HQzaujJ6DnMghniUvQVMdfmcx6M3hg5tH2P\n",
       "6ThxY3fFMcExtztUOd0KWKR/t6r9xq2E6yDomVfZaE/qZ/TzwzXf1z0QMP+iDCynIVBQQCGvfavb\n",
       "hU6LtjbAgyuzT/DDTHcjd1yItffFiMYNTup1x/r9O2o3ur3976SQC3on2KEM/p7HbhMeZRtRF9mq\n",
       "eSliCqYNCcc3B0PTj1A7hfgBlBGPMnCUZIZSJSB5Bitt/hSowDoBhzULn3jMstPYaUP+KQ0A3spo\n",
       "9rj2r6qLKq0DzOlSIGNXwfAVE15C1//BoCPsVyBGxUW7Vt0gbGGFmZvIQPokGaRgISnI/kwnmCy+\n",
       "8ZiG396/3a/UXDG2Ap3WSshWOdMd3j3Yd+kAxDjb/aDk/iDFFfog1y0AAAHIQZqGSeEKUmUwIJf/\n",
       "/rUqgACQN7iAA3UGfNVmHtDg2U+xZqXWHnYAAUOjSpmAY03UM+AxDIRC27/YMRBUM/I3re4tBRyP\n",
       "4XhJ4x1pFKYYuLNnqDEKP/yEkOBPxetwZFjZUSivkHUeMKgX1ipf5V/XphYQzR8v9VxnldWOYKty\n",
       "abrz3o5ug6w9tof0cFPeKhNYSmWN1qZmSxMk0n61o0WntH/9co3FJa8Pp8aePoQUiTvqK62eCjQq\n",
       "0OrPLgDidNIksoRa4ffR5dyS5Q3Z0Y1CYq329ge2XhIz9eVjAq3D2MNipwGqpLiiZj4GKmv4fcDo\n",
       "I1E/enFu6aSgJNUSarPVuW/s0CWGZgriaCNWwdQAbQbD2VOl5O7YBGeq2aUUj1gedmFDxeALQ1Tc\n",
       "b3sCQ/lsq3LGxsQpFfRl0QyFjxfBVDuqXgMwHM2pyS7Bxsu5maVNHC3lNXKCTMWYhiJgLHDZ7GF2\n",
       "fP/pdLZ6pFswx6ud6wDb2emRMBfOF1BaCVYS2MNu3UNkIRNdxzFBUSfrJfh9n81jMZBbLCYMPFnY\n",
       "ADr2Lj40+T2IWeG7Ksh9OBKfo4nNSPZyHSQCxpST8wJcABkN7BXN+hG65ffhAAACA0Gap0nhDomU\n",
       "wIJf/rUqgACMJIpWajEn5goLmlPACUHTbRzTQ112LfCCQ/Lf/kP/AEfEaHbXeV+u37lB2Q8qkIR/\n",
       "BIzTHu7t8MuBh0jVNgrZ7vHB1dlHVwu2AvjCe9S5p7iwO+wXsQrAtF5Eqd4A4J8wzcddA1l29fCa\n",
       "Ev+AkyZPALJqx+JVsYyRUTv8lll0dYD8z/tQTm2QngJrge5mwebo4eIsssIuPQIv8EgeilybjmV1\n",
       "Akzj+KhuiePmkTs1B4r/xzhilwF+uH6REuGrgJ3wWZZdWpymfw5QAQCJSUJxYV/A8oqJrUdk/DfN\n",
       "TGVWy/a6ACgJjTaEujZ7Z6kW14daFg6tSOZrk5dW5N2txFrqLNvEfLcCo0uRCWagbg1e4YLt9i7v\n",
       "jWidUgE8pTUbOBjn8XjFMiyWCZynMn57wXE53MYn5QvnstyCsTlOiLPQQ4TezXXx7+jtUukkkkP8\n",
       "ATIRZ8LuqlOwHnhi71FhR/K8rkkUQXZ0+p8ifmkk2TW7nIckWI/YgxKy4qevSDIFFFYu0G/o/R4c\n",
       "pYmQW1mhwHQ06uajHwQ23/e8VoINitYRWM92vFcr1sl6OFvUVjfS6vdfVWI6wR49VdNfvxmmJuU9\n",
       "uCJhlfIlG546E1Ow3zUdqicHOYQwM6R+GpxKt51/8nPRLLXLgdoTamw/UHJ8Ldy7oIcFAAACHEGa\n",
       "yEnhDyZTAgl//rUqgAB6vmTlYzf0H44AE6ANSmvtkA97YE3WS8f/yHFz+3J6OU+xHPws5vEoQj8D\n",
       "hFHwtm/GktNUvxXp5sMelrbgX44ArBsQot29D6VGOKz5MJxRs4XpVlXvXWlpoReRhk4PZhxzSCBH\n",
       "dDW7U0cXUjgGy8a8/2OkCsn1awYO0fvvr/N5FmSUzGUsu2wMgUksrbzf/SGUUJGRpylf3k9UeG0O\n",
       "63ifLPYzhHv2ze9HpI99rHyU0pIbnk/HgXeMxDpxSRTSacm2ZQgMtui+TCh6U1wGKEpIcirmRx6X\n",
       "d2x2zdr1iGm+kkSpJMC8IuNQRGAtIWvjZB035VD8wifIa3F9j/vv4Z8P68FuEbJYLKyeA3oWrfbo\n",
       "vLZgKLtow6U5QrNdEMAsEfPCNnfPOqilwgY+xP/WNXksObkTOEyGhwD8Ei6LBfirBX9yFj+lUXQT\n",
       "UkGvj3LxPt1roXhQohweKDa6b7sFNyQ8nzmO4yNZ1GZ4dkjhhL/jTjc+hluMfsie+0/8G6gPtGro\n",
       "fedeaYbbpq427W7JESRh2RG34ZFjypDa2KvzuXHbJhC7ijByGkty52ACcZm1EwH+2fsyOeq5Xx/B\n",
       "uVOgONAJACEmayubiPvk2gme2fUYIo3kiFTgZtHyunxEazLMssBFjZ3/nhEPyTQ548L0Oyy+koq7\n",
       "75sJrLXoCu5lI3P8YjpsjfHhWF2byiAQ8AAAAPFBmulJ4Q8mUwIJf/61KoAAb751jkMQy/0/p+cL\n",
       "AFgISB39NrW04ypsIYJn9TRqszbtFVcvFr/OdJtqd2dgQKnYH0u/rDq9mdINCRwiQxppnTeqIy/J\n",
       "PayugsadjrWwsAM4+2B8zAghmRw5PppaooF+8CwI9xb82Vu/T1TPVUOmaVMzq0z0cvFsX1qgHdpi\n",
       "ARvIdP9XfplJ2Fzvd6QF1a5PiMJf/hrD1Jkz2Xw9LqF4HoYx106bbzCup3D1Hp857PLA1sCFedBU\n",
       "FyP3ZE362ROLeLY9LeUngdWwMDPecgZvJWnd13usC4qa0mkfVQBLwDpgAAABSkGbCknhDyZTAgl/\n",
       "/rUqgACQ81fwgqeQEJcckHGIQlFlP8E2QamaA+yvYFP7m3e7q2iMGsl0D6u2RB2PPcko9nkcOLhh\n",
       "QtcSqTg07BxDmpvC4hUb2KsAaerl9/nMotCo768GWebjghntQYfAraTvPL4dbXEKto7B2ImW03za\n",
       "n/bqObC2d2PFRgb5aM9YWX/M+72d6kJ42fhi/QXljZ1f/0f6WFtUEGed1omV/AclyMC1m9Dta5BX\n",
       "NSvL+D3MhMKdTRScWdsnkt97ILkjAPbWhXXm3B7Gd7Noi7jzeSbg2QzWEtnOeNW/etpalnnrmnhI\n",
       "BMARxJY3if7E3ZCmrstDDS8bBi2VN7/5f2tAxyHZvJ0KMKmphQl+KB//j3qpHZ+brQeG3VqYmdTQ\n",
       "AhMVvBrmkjqffsdwdjGogJzRC0Q5Iw0/IVJ+r+bXEaACLwAAAYhBmytJ4Q8mUwIJf/61KoAAkDe4\n",
       "gBQm7+unHKbc+UOd0EIAHIs/PXUBx06GFElnDso8OHj0DBTmunUj9lFos6I4i22oQl/ujBc29LBT\n",
       "i28+mmf8RBKe8EVP/4wmBsMDKBK0Jzgc76W08l2xaqfN9yD/r4VGG12sP/ajUREtT9f4hmjhivo6\n",
       "wY46enPE0VqWeGn8VoDOzPpBm7xjXB/PgOp80YMXmYP4X1IVN48hhC9KfLo1vTDjdkewDZthEnCu\n",
       "G9QuolDdnmDph/sVTV1XzFF/FRFJRP8ZPqueus6j9fMYHOcMGC3PnqzvCPPEbun6A2ijpfjLcPE3\n",
       "X0yIhzG5xxCLcTdgVanuRaLEY64exbfYQa8GBZHVqE+2eV7Z/sHKCx78PC2R/35d8qdDCjNgHm/V\n",
       "2OsIQ4ONRWtQH7S6aVXk/HA7lYtCXV6v+taqnPO+S8h0LKjmAXFMwyNuud2+OzrNnMok8PTEwzGf\n",
       "fTyMnG5699nmn2vZ5G/03fuJkPLjheObvOlfnQQUEAAAAaJBm0xJ4Q8mUwIJf/61KoAAjP3Xvy4b\n",
       "k5E7NA8SSzIY8SehhfgCOVTXcPX1XgDnlHcZz/v/4B7AKhPK8P8KWCBhKC57JgjkuaoFXgdP9uGM\n",
       "Y2xriyulS++feWL1SSDr2IOp1B8ZXwSF3PejC7/Nr/G9mtLHh9xl5T55RaOLbCpthVbRp+NBGJVf\n",
       "4jSuQB8d/LQH/sds7T++tmLXG6l2BFEgZT8FlpJMK5RBHcpIOiqQEa2nM0HZ31/eoAepMlHO/d5m\n",
       "stRI4gA+hw72vAPBEZLJhr5bN4HeqUA7MTCbThGO1tsd84ZHxkbFXg/79LAvUXG6vh79U80aI01D\n",
       "SbojCzTfhXoIef/tcf1K/1BUU88Vtc+xN1qMJA5iMJ81Hvp1bplHXr39Njz/TVLo+VNWvvhF44vy\n",
       "xC3DR3QFM8lkTsAA2Kik2Wg5ZLtyGx1DWJRfPD6+rxLq3yeTbywSGlMFzJ4RWwB9Xpa+Z9aczQaB\n",
       "lgZqj6dicXEXq9bJYe78ryu5jW83UKHoZYpCbK8Iry6ogNnX1qreR6pDy8e+olf+rUpbnr5gAAAB\n",
       "VUGbbUnhDyZTAgl//rUqgABnyo5+2AI5VNb8wyHbFvtxpV+kPp3cvbNhReMLMvd8KxVOd88SKDKv\n",
       "WsZ63A3jYNREn8RVthQ/Mj0uyLDO4nKrsmPLOxFQ3ZzOg6MkaFVIFn/ufWBT6AVBkB9ySLC4/82+\n",
       "SGir36x+894OIMN7AAhxYv77qlTM3r0D/eujb2usRlQGtt8Nhhs3SOwtGs4oNy6giL+3sCSefqRs\n",
       "Ozjl3VaetiV6lYW9wNvIpZP00t7vWooD5SYXmRVCx1rpCdLZat4OPQlE+3bVPFrDNhi8mhDVOhw4\n",
       "IwA5zh6Kx4NXEgLQZINXmR0ydUg6uIx+L7HHLlecA2DX0hTAigpx5QBFL1EVp9p6hwJrLcXU8N+o\n",
       "ghF4vMfL6afyUlP/tquKs+8i9qzXgXltvJgAAAMAB33Tnhlvl11RoniQXdfjA8QdHTnnorIAe+V3\n",
       "AAABX0GbjknhDyZTAgl//rUqgABn9kRtwarIMCAG5niHFGRqQxFjt5BCDJjOT/EV+gdaw1EnXvMZ\n",
       "HAInwNEc81Z/QSMOGN/nIiY3+3obcsxo9U1iTryVaqKkb779qL/+c0YJJkagu88miOlQlvsvLZIr\n",
       "l5lf6MLmJELMvzuyYpTrF7daqmkbuXhkOgIEjaCYsInlh90C2m58g6IGubC7J4YDi4N6P52LTu0l\n",
       "DOVzVJ3bi+PF8dWHv4R/cB80Y+uzgo3T9dj4/4GTmR47wTFzbmac6LjLR9uQuKXhvrUTPFjk4YdB\n",
       "i/Dn0ZXOTO2yndB0FN057GKhdu8/7zxbVB7KlJajfFSvc8+GFAqLnNdaJdWK6jJWxW5hI5dlXhQQ\n",
       "kVr7lSg0OSWjlSBuC/v4yfDZ2bwAjJ9GWcZ+py5pXIkm/ythWaOkECMgF9oOQCnmu9xKAC4xq5QL\n",
       "zxsv+8GdwDIi46tpGQAAASdBm69J4Q8mUwIJf/61KoAAJw9SIJluiW3ux34QAbKMOAwH2Wc6dAIY\n",
       "D3yzxiuGG/5DCLcLO/PwEolX0crVjgVIsb/GN4dnD/IIh8a31ZXfdIANoh/CMWRINNL5BMuxI4et\n",
       "9FDNue2kPlKR3f/xFT6Ccp4VKoAZ9XXZ+DfotIYnvKRnz2l3QUzv93hGdc18M8udtqRxeRH/14jh\n",
       "DQUmPuee0g0+/jNH1U0IPMrilZWpGqdNFKxghZmrgEs8JUciN1ils398rO4vYOCQjiv+8uqiRGNG\n",
       "vWSyPvbv83BhmPxzeP7gmv4Dlf7sddmJP9uCnypjCdGDtAkYucVGoIpn1AbUjfHJx92P3fAww54e\n",
       "0SZh+AF2vuCBDKvOjyBpwRrrlyWgFaAxfM+BAAABqUGb0EnhDyZTAgl//rUqgABydx/8gv9cAE7q\n",
       "PiNXpBfOWeH9YVt/OcJ+gb+DKatOaN72r5tLXD9ZTqn4m/rQkP/+ExiPwwklYmIfqIqv193NpNTR\n",
       "iiW/3ExXdeKkeT+mB+i4+nCBxklILuI5wICBEsG67eXZvTzxJf89MC54/+Rtulc1SO/WTtkSAr+z\n",
       "Vv1fUR6TCnyE4QYbPNgC/fw1Xa1e/teJUTxTNmmbiCDsUG14dCvgjb3rwhDlq+rRJhn7C/XguLj6\n",
       "wgjltCmkgVgZuf/+aI7Y46ktKxVkjr/MmxYWtcIEyT+hAbQPAK8CeGN8UGXu0Cy3MgCnIIbeUa+Z\n",
       "eeJi9AfbMPPC/q9QLjY3dBAqnwCnYuTppCx1Gn3Ec+DHeNQN6Z/eVtuLLLYoBLl5AZJmL5GHfe1Q\n",
       "TXTssTd/kNkWe4kaOnJ6aD4DhB077wMtGXCU/cQR+OXVeB0T6HOd8+BNwgyNRMPPDohEXKdHCafJ\n",
       "ZR3nTstqGywuBHsubeILmFaANt4OxV3VuIg75RIWoDxrwU+0wOSQQbdbnoYR6L3+5c6D7XqDUJQQ\n",
       "AAAB0UGb8UnhDyZTAgn//rUqgACANX7rsFwA4wMCg0gl8HeOIZ7YVJs84zYa125+njBAx9gy+fvs\n",
       "e96yB8NGDmFYxQbhrCpjDolG4NvQBu/4hV27Hc/wyX8hNgXwhH2f9fo7d39w5kccQKurxjs9X5GA\n",
       "YiklK7N6C5SKrzwl3Dl8irJWaumn6HUAm+nFHR13zhUMctdRE+gP7AZKs0la7EQitnG53//kAO01\n",
       "XR1CwHXqaehbqM6Rri7ot0FLS0oADCaQuB4cLxUUaT7ALKVY9d6ofAPmN6ABXaiwE9wMgmss+93a\n",
       "lTojOY7jnfW1npUFh56LiwyE9Mj5z9R+exBV8aOQD4UZp2/A8YZmzFEA3JNeWmIMZo3bj0s9gmTc\n",
       "Qu3VqiGxjkXVeYAhbhVKjFVnH/8B8H618neFMsHo0yrBGI39DTAT3oc8YibGzhOZWwZee+grtTyE\n",
       "GnibX9dtHvHRqr9GYetobgJ7VghAIeeGWP/xaa3aOxidiuY2qPd0DKWbBlHScaPdaiFIPezTOnZC\n",
       "KquJsCkdZNJxdsFCD8fGWeWSBE5dRkMsG4kTl1/aUUFFRuF/cmruCxJ7rGBpSNqweZyOn2V5zz15\n",
       "1gpSSdfQObIUv4Cb8gAAAYhBmhVJ4Q8mUwIJf/61KoAAgCGMNt9xBVfSqNEWEFzDiDWSyxz5IH+Y\n",
       "53qmRiChLpvENvIFyEbhUCyAKwtyQXjYWHsonMBl+OoemsSPc6Y5JuVTV3yBlGH6UZlbeHK8QD4t\n",
       "kj2X04NddAszL4xBG/VGJSNhPX/+sZntcj7N7801XppOsBPwduYY0l2u9+7vv/97QL9VhnYsgtip\n",
       "FLsUPvsSEibDxGIMETcvwuaqjgOK5Sv+T5KdH44FTMfxe6czEfrcALSneU4Iw522pXoqpSw7aBIu\n",
       "O1d5Svk+ufDbK6nZkOatgX6ANCHlHpI8LGkZcV9FXCDnPLc1X6ZVpNWXLtkDk2yktL4jCcQ1Vam5\n",
       "zZjIWB292lCpKnF55GPxEIBlL296ktdiv6PimjlT9BYoED2cYJsvpLa8KuaoLrvpDash5TrSeHDH\n",
       "k9ZzC0ycD5uMHizUitm4ScO9KgQFcCvaQCgPGTkEs+8cfY0cP4hNyyZ2PtfZqI8qBoOjKqp/5qag\n",
       "8R/7HWKXVhwCHwAAAYNBnjNFETwQ/wAAs3yI7vi4AOMK5R3HXgFLEJqYi5EFDgLcHvDaVzffsgnY\n",
       "+SxiP28H2droX2OtiNYt/T9JQ/5bFn4SlJUw2RGsf0UJRsZXf6cVfQYhQOtWIAce7cJPX80+9dfT\n",
       "ZniJSsFLUdAS8h7f29zhWUutv2XHRHpPwDv1/+COBCloAuFx0jE4Q0WFIMn5XiMpFW6HwMNSLers\n",
       "ZUjkzjFUH+aq9OXsc4S/x/0///2REhwBJr9wQqIrRwMQ4NJMBvVZcGWki4AVm7u3UWeLVaPXcS9o\n",
       "jDM/SeuthSZ7LV0Ub3Y2CYXG/StENLLen0uRBuOuEPoPkt4P0mQFdp5XjUGV3k/92pLBOrSuC/I/\n",
       "8NRi7z/COxH3LZnYIqDAU262/DrJPr9q3tOITgV+3bV5aGh4CkwfRAjzjfHuZUlBtguTtsW0dWXx\n",
       "I5+YscEdGOMvl+P3/oWHvvFEfN9y7t7KqmNpplVEToaerWDUr8kXSM4SFkgepNKC7YQC4sKuyGOi\n",
       "wUkAAABWAZ5SdEP/AAGRux9AB/CBgAHzT4MCbpoYuM+PMe7ZNugrOCl/i4cnnp7Yk2EHAz4mc3jR\n",
       "tDzuZKuLMNdCgeOwAPzqXqyrSNUDVarJJeuVU+NwiZoAEPAAAAGYAZ5UakP/AAGR8nHABNNlSNFm\n",
       "bcGBivuObr5nnHi0hpMZ4QH+pKWteu0c0JBvrEejkjUFzgVK9lPqgvH1qTwc474Nm+rC20My6mQj\n",
       "ps9m+oJ4BhA3L3sLcVdSyXyFh+cr21lEjNIbIwzTwMAc0TEJ3P/JjXXIsNqjk4sr0x4929Ebo44Q\n",
       "OOZrj5AN8OyGLpCtEyVOevZICdr7rgH/JOQkijNrzCoYWz6hKPOtVubV0FC7BkJVsKpIkINuzgh/\n",
       "YiYEWE13TXAGKhbl2/oOkXcZnxr8dNxobzNDAyoeElBcEz+jgPW7ClmHR1+1xu/0iYtPNU5FD6ux\n",
       "bq1r9dcN2EYGV6/jQHa9OZ0Nc3VvBUkKP45qiAcCchXUOECBE/UZEqVnfG2P1+U/lBGan3gJkYb8\n",
       "JI5UWX5ggUFca5ZIWXRyZ9BAAKToeZ/KaIh2hjecmvZDPErU5fFahcQij2NSLlzbL3/G9EOrGCNE\n",
       "VR6yIv1sfqshHjgdXyZgV66C+vwmiu8PIX9hk0NzHdHqXkm8bFxM4JWXtbg3PC9hAAABrEGaVkmo\n",
       "QWiZTAgl//61KoAAgCTpFL2BKqc4as8Ec5wk6o3myoBGtHSLzGHQ7RamKhv6n0uFIyEwAXtjt+2Q\n",
       "2H81o1dsDl9iyhj8jVW9ZuATVUe21ceJP0RPO2VvAyYl1G+C40iT/G7FXpALebhK93NhQiHwITf1\n",
       "E0svg658d70J3hJx3nG+zEiIEhCS7fdRf/xwCUfPIliVhIPaab802iyL7vYDCC8MBgEmAd5HZzvc\n",
       "hl87dFMqOxMXU8uT5KYhDUhVpsTr4eoOU6jCJOgw3SbK+7s3f5HLtqoiRjviTB3riu4AdEYv2AYE\n",
       "7UYijN8kpKqwcGHGjkA0x4Wmd1cU8GIPVY0CvE+6LbvDymKZLHZ9HX7G/EIpKV0/kk0S5ozo/tTV\n",
       "MbXAmoZ3lgfPBWWffDWqxCQXW7bBgK0pt16t5FieZozLTtPzVqlz66VnSSB8NBFFnUlf6P2LQfu4\n",
       "K61TNd/LM2mTL8IEKAUrb72zW4Iyqo3OuWzha8VCVnP38D5eJxL5wJE9K8tI03KUs8J6NC0KRjb7\n",
       "GwxeCoox99eqhqtlyfj5DAo1zMK9eIQsAAABeEGad0nhClJlMCCX//61KoAAkCT1EAA5tu/rpxym\n",
       "3PlDndBCAByK1omwAxlqPdWxr/j/+g7LtjKfKIA1W3vVS/YrQUKb/Hy6lKP2WHT/qFsQKt/8YQCe\n",
       "Jog4k3suoOygvAmg/CUPhlUgs7DBuijXCyVgzxJxzXiaDj8jMD76P581J9+6h5VGs6QwXyO+gy9I\n",
       "Uryj/s7k0E4iDwLZKdwGLv+NL5qBXT1a4fYwWiR25DTI22HlHya9BRYlxJPs3BEbTt/JdauTd/j/\n",
       "thNXtUvx2UEjAoSE9O9fM5gD/o+Hbw6LkFpc2pyUAM+JT8zRmb49erHSqUDsH3sM5ZigTB9UE06W\n",
       "WM68C6FhiAle0mQPPQMQ2st+HaQfZoV2M3db/1u4SumSy2j/GQKAz+f3gWMIAu/RVb84mDx4l32X\n",
       "UWOhBhy8cKwMj0l6Et+JgBktWoxotxOQVgbKcvXj/VkqO7myyE2iLm+EsohIX8L5amaeKzhhm2TT\n",
       "5ORUA4EAAAF7QZqYSeEOiZTAgl/+tSqAAJAhjDbfcQtwixOeD2uGtjUCzG3P/I8KbABf205vKmWa\n",
       "EQ2C1v7ThIphAAQYxFMbLwvYbgeu4HflawlfUKPEAm2x6BYONTyEE5KRDWdXCJ7pguAABfSXBDDF\n",
       "4gYTunLYGBDNOSedR2/U6n79Zjs/3NWYf/4SP/p58NgS9OXJMSG6gE6mryF4/lx4ntg/5m+z12uv\n",
       "TjosyOcuRMD+3cHDGOx3eUq9bHEi8R6d3IClYdUPVSsWSW9gGcLf//dHBIWweeJXssZwP2Lv28HN\n",
       "X6kVPmhT8rtH73evcuTtGJIp5fyZ4w5GilmUZmeRoBvBoH/xxZbBGz35SdxCbJMljDbsCRJm+Rrd\n",
       "141hFrY3oYYJ4Y+3O6Oubq3cACPimwG75JciQHRh8UHgs2S+jD0+v+ZjsMYGqY7uzM287lNMrg9B\n",
       "VJA3FacXaN0lM1ztSDD5fRTjPLMDh/HYuE8rc0nPUCXYAs9DoIFrI79tMEgMWQAAARBBmrlJ4Q8m\n",
       "UwIJf/61KoAAZ+2DagBXwHbfmGTBjHBjHf64qa1QdvQeVcWz7BOs6zfuI8I9+Te2m7tcEfSi8b+k\n",
       "LCJkQ5rq//v/ApcpKqpphdEHB0oolSKXhnRvxkVlE0UdIidpVz/F2QMpPUuHqre6aMQrEaasUOA0\n",
       "PLwn3f64no0CasHeYcYblHKsWwkrZDCdYSCq2KBGxg4b0yLsnvjIgm6927J0jkcHm89v2uBAKmu7\n",
       "usTO0ReNoQJQEQbCzK9TEWytOsxltwes50pTaAs/1lCvGHy416duhjV8Q34NpGRI7ZOt5sQgnn2A\n",
       "M++COjOxRgqfI0OtGntJ2+anJkSo7bcl166S2Vahb3gYxZoHxgAAAsFBmt1J4Q8mUwIf//6plgAD\n",
       "fbP/9CW24AW9dRjzOd8aSky9dL3XYIFYin9A503f+K1ZmDm5R4z4TFbKQ7RhkrJz5MitEernUz6B\n",
       "QXy3+0J+CuuZV7datE+fVrVH+GzqOFRFRx62TLQu9Hm8bqsl7UEdD5KCupc7NVlJVAIZ2B0Pmxnd\n",
       "43LXqa5+iYhdOHXsZZJ+t9NL6TKhTwxy+ZFrZvc/UHlS1xsoTN/q1XHv7ZMKNedhZk6O5Kmj70wx\n",
       "cxN+6UeZR4VMA9j5I85pyONvIAP20jxJCG7AEwmigf3/maOPr6/+COKLPqFztEeiHUzadWdn7/hJ\n",
       "yeh0kSEQE6eFO7ouDdSd/AVgFZ6jOIJKdZgF/Zu+ntdOp8AGnXK2YtOIbbjpLSdDrDBtP/59jIQv\n",
       "1zibWIHSCCghyVGqkly9KjNn4BvUk16jQaEkXqXJoa2q8PZbYA/vQFtRGRYB7Qee0QEk8ylV55xC\n",
       "G+nTL6JS+BnLSaGTWWapaVfiE93G0CMfCD047y/XxDAMkqoKiCFa2T/MMA3COT12GpECuRsAYq0p\n",
       "YDjOqiZFNyjvB9qQmdq/z8fv/DqtEWN5Lr1E3hxggWylLtmmaDmRVbnOEOr73v4nszxOS7LKkIag\n",
       "l1NMOG6/mZfGpjS14pHdSwBUQMhzbv+PF3jX2/RBrE3kg040emevpN6uiILqsxxgZHrYkKP+fnHY\n",
       "DnmffGIOM3yT/7a0TQia0/KQmgDqPLX7P9+86Nqn5DH86xIdIvdkBrNF0c0mc9eBGN7QKXyJ5vsr\n",
       "q98w/P/KlmtgF1uH/nWfZJf/Y0Ifu46OG62BtGZfuRbp+H585p8+HnyBKKf5QmiUvJOZLfCrSlye\n",
       "b+YDAvmIls2igiyluNWpJ+UBDioTlcd1sRSB6zQtaBU2SKVsHKu277kTM78Z7d+Cm6e7zbt/LrHb\n",
       "brBTAx8AAAIGQZ77RRE8EP8AALVt+j8AEso+TaOqMxgzZL1IsZpuwpfrOM0+x2jv/Q1umUt4a07N\n",
       "/8nNncTc1zIx8o+T3UkTvm3KikgCsjpTqCAfQp2gNNfCzUJWVQ/Bs1o4wsRYIuzKidVu9iC1d2pE\n",
       "4luHoPP2iZRyiXDqh9U91+0IxbucQzsAdZP4fLfvIEjkJCxyEh7EjLwRxXPgioIaxi/THOV9Wyso\n",
       "Sx9Us54RsZnuQdBn1pM1OFZr/HF4ycYcimvBv6y/aAKE1SvVUy2FxjbhjumQuO2foBJcYI34IxDc\n",
       "Mcym5SLtPzITlUW2sBhUyv8LQ8Bg+upqO6sqsTkRprXIB5tGbEmjy+ia1Z8CdJdmgcPW6r6ExxVb\n",
       "7V08SaE3Q/wVA+4kc26hkhk3bIW+zsG+XqOEbabOjJ/skgRg5+Vrhta/2uzgDoNMV3vD1lUG73+u\n",
       "vAQ245VsYdArNq8tVHC9p6gqFQPh7A3aRZrbrIzl8QYzve3VyzV98UlsflceEt4evJRA0t6NrB3s\n",
       "QC2xB3Sp0xIC79YRPvhHiCE8Q1FVzibmJCfMfM3+7+bJGyj91LDh9LO4UaMf/cHq3jRxsgywuCu3\n",
       "Cs9o6WkbPSJ0AJ1X1wq76KoRfcTvRvPWZSXPsttkA+r7OT7wfEQ0SpRtqO0olUcGwfXe3EEu/+ru\n",
       "/lON7W1DptYvCFOAwIAAAADUAZ8adEP/AAGR/50mTp3MhnbSTP4RLEv/HCIKegVECrqAxgA98AsM\n",
       "ah6L/4KiewgKGCcuR+NiKTz66ctl7flBN5iakGnc1r88oywT8SDGM4ng7yfBuPnyD/xh4ra+tjQ2\n",
       "nbeT5P7fe4PfhNEUYRStOpP181apV4B2WlvJf2lM84Vx08EwhHC0V4VzP4ajsI+FDQNS04ek2sM3\n",
       "m4yyVZMkKOMmtfwScSbnc0LnEMhIBXeMlHnnsXJNAI+bcaOMPfz5gKtDLX1FnidwSzIQ6N1CdnnA\n",
       "N6EAAAFjAZ8cakP/AAGRzOXeBYe1zUVTf1ygsHcxSSfN/FOq6sDO76yEHCkpI3L10mH0hBYAG1Tq\n",
       "i5aiv97ltgjA+SmeTWozmxUOnmY8qxLbF5RQT0oHirB5mBPNyFzpLAY2wsJO/uWmicm9J9oc3GJH\n",
       "celJWGpcD9fHBrQqTlZYD0PCzb5lTEIBe4a7+LYr8HSRSGK2kNZHtbAkw65LD1mMD9i82hfKbj7U\n",
       "oQHniOWxaIMAmgAnm9D7sKl1fDBnCoMjW1DV2PT6GOYdFFY5oRqC6pUIntBYAdn1y6laR8Ga98hB\n",
       "0FtpKvMnDK6P5lN8emT+QGhGe8bkM+yFZGau+JhJfs9yN0h2NMMoCT2nvr/Krc75ynrIN3fu9YjI\n",
       "S8CviSsYBm/zX9sjseaEPDtTZjjNHENH+zonVmTPFpr6hOczFjCw0E85bqVaXG2P/bnMQbWCo9lN\n",
       "U94G+KrJNMhLmOLyF16CGr5ETQAABB9tb292AAAAbG12aGQAAAAAAAAAAAAAAAAAAAPoAAAXcAAB\n",
       "AAABAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAA\n",
       "AAAAAAAAAAAAAAAAAAAAAAAAAAACAAADSXRyYWsAAABcdGtoZAAAAAMAAAAAAAAAAAAAAAEAAAAA\n",
       "AAAXcAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAAC\n",
       "WAAAAZAAAAAAACRlZHRzAAAAHGVsc3QAAAAAAAAAAQAAF3AAABAAAAEAAAAAAsFtZGlhAAAAIG1k\n",
       "aGQAAAAAAAAAAAAAAAAAACgAAADwAFXEAAAAAAAtaGRscgAAAAAAAAAAdmlkZQAAAAAAAAAAAAAA\n",
       "AFZpZGVvSGFuZGxlcgAAAAJsbWluZgAAABR2bWhkAAAAAQAAAAAAAAAAAAAAJGRpbmYAAAAcZHJl\n",
       "ZgAAAAAAAAABAAAADHVybCAAAAABAAACLHN0YmwAAAC0c3RzZAAAAAAAAAABAAAApGF2YzEAAAAA\n",
       "AAAAAQAAAAAAAAAAAAAAAAAAAAACWAGQAEgAAABIAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAA\n",
       "AAAAAAAAAAAAAAAAAAAY//8AAAAyYXZjQwFkABb/4QAZZ2QAFqzZQJgz5eEAAAMAAQAAAwAKDxYt\n",
       "lgEABmjr48siwAAAABx1dWlka2hA8l8kT8W6OaUbzwMj8wAAAAAAAAAYc3R0cwAAAAAAAAABAAAA\n",
       "HgAACAAAAAAUc3RzcwAAAAAAAAABAAAAAQAAAIhjdHRzAAAAAAAAAA8AAAABAAAQAAAAAAEAACgA\n",
       "AAAAAQAAEAAAAAABAAAAAAAAAAEAAAgAAAAADQAAEAAAAAABAAAoAAAAAAEAABAAAAAAAQAAAAAA\n",
       "AAABAAAIAAAAAAQAABAAAAAAAQAAKAAAAAABAAAQAAAAAAEAAAAAAAAAAQAACAAAAAAcc3RzYwAA\n",
       "AAAAAAABAAAAAQAAAB4AAAABAAAAjHN0c3oAAAAAAAAAAAAAAB4AAA3FAAACxwAAAWcAAABaAAAA\n",
       "7wAAAegAAAHMAAACBwAAAiAAAAD1AAABTgAAAYwAAAGmAAABWQAAAWMAAAErAAABrQAAAdUAAAGM\n",
       "AAABhwAAAFoAAAGcAAABsAAAAXwAAAF/AAABFAAAAsUAAAIKAAAA2AAAAWcAAAAUc3RjbwAAAAAA\n",
       "AAABAAAALAAAAGJ1ZHRhAAAAWm1ldGEAAAAAAAAAIWhkbHIAAAAAAAAAAG1kaXJhcHBsAAAAAAAA\n",
       "AAAAAAAALWlsc3QAAAAlqXRvbwAAAB1kYXRhAAAAAQAAAABMYXZmNTcuODMuMTAw\n",
       "\">\n",
       "  Your browser does not support the video tag.\n",
       "</video>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f06760546a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Trajectories can also be visualied via the simulate function \n",
    "# (note: function buggy on some systems; needs to be run thrice. )\n",
    "submarine.policy = policy\n",
    "HTML(simulate(submarine, [3,1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·'],\n",
       "       ['·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·'],\n",
       "       ['·', '→', '→', '→', '→', '→', '↓', '↓', '↓', '↓', '↓', '↓', '↓', '↓', '↓', '↓', '↓', '↓', '·'],\n",
       "       ['·', '→', '↑', 'X', 'X', 'X', '→', '→', '→', '→', '→', '→', '↓', '↓', '↓', '↓', '↓', '↓', '·'],\n",
       "       ['·', '→', '↑', 'X', 'X', 'X', '→', '→', '↑', 'X', 'X', 'X', '→', '→', '→', '→', '↓', '↓', '·'],\n",
       "       ['·', '↓', '↓', 'X', 'X', 'X', '→', '→', '↑', 'X', 'X', 'X', '↑', 'X', 'X', 'X', '↓', '↓', '·'],\n",
       "       ['·', '↓', '↓', '↓', '↓', '↓', '↓', '↓', '↓', 'X', 'X', 'X', '↑', 'X', 'X', 'X', '↓', '↓', '·'],\n",
       "       ['·', '→', '→', '→', '→', '→', '→', '→', '→', '→', '→', '→', '↓', 'X', 'X', 'X', '↓', '↓', '·'],\n",
       "       ['·', '→', '→', '→', '→', '→', '→', '↑', 'U', 'U', 'U', 'U', 'U', '→', '↓', '↓', '↓', '↓', '·'],\n",
       "       ['·', '→', '→', '→', '→', '→', '↑', 'U', 'U', 'U', 'U', 'U', 'U', 'U', '→', '↓', '↓', '↓', '·'],\n",
       "       ['·', '↑', 'X', 'X', 'X', '↑', 'U', 'U', 'U', 'U', 'U', 'U', 'U', 'U', 'U', '↓', '↓', '↓', '·'],\n",
       "       ['·', '↑', 'X', 'X', 'X', '↑', 'U', 'U', 'U', 'U', 'U', 'U', 'U', 'U', 'U', '→', '→', '⬤', '·'],\n",
       "       ['·', '↑', 'X', 'X', 'X', '↑', 'U', 'U', 'U', 'U', 'U', 'U', 'U', 'U', 'U', 'X', 'X', 'X', '·'],\n",
       "       ['·', '→', '→', '→', '→', '↑', 'U', 'U', 'U', 'U', 'U', 'U', 'U', 'U', 'U', 'X', 'X', 'X', '·'],\n",
       "       ['·', '→', '↑', 'X', 'X', 'X', 'U', 'U', 'U', 'U', 'U', 'U', 'U', 'U', 'U', 'X', 'X', 'X', '·'],\n",
       "       ['·', '→', '↑', 'X', 'X', 'X', '↓', 'U', 'U', 'U', 'U', 'U', 'U', 'U', '↓', '↓', '↓', '↓', '·'],\n",
       "       ['·', '→', '↑', 'X', 'X', 'X', '↓', '↓', 'U', 'U', 'U', 'U', 'U', '↓', '↓', '↓', '↓', '↓', '·'],\n",
       "       ['·', '→', '↑', '←', '←', '←', '←', '←', '←', '←', '←', '←', '←', '←', '←', '←', '←', '←', '·'],\n",
       "       ['·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·', '·']],\n",
       "      dtype='<U1')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Lets plot the final policy\n",
    "draw_policy(submarine, policy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Question 1.3: Policy Evaluation\n",
    "    \n",
    "Define a function that computes the state-value function $V^\\pi$ on this environment when given a random policy $\\pi$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Question 1.4: Policy Iteration\n",
    "Apply policy iteration to solve the submarine navigation task. \n",
    "- In what ways are the results of policy iteration different from those of value iteration?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Question 1.5: Hyper-parameters\n",
    "\n",
    "For both policy and value iteration:\n",
    "- How does varying the discount factor $\\gamma$ affect the calculated policy? \n",
    "    - Repeat the same experiment with at least three different settings of a discount factor in order to make your argument.\n",
    "- Given the insights from your experiment above, suggest a suitable strategy for setting an appropriate discount rate?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Question 1.6: Noisy trigger\n",
    "\n",
    "The mines now have a 50% chance of blowing up if you are within their direct vicinity (i.e. the blue cells directly on their border). \n",
    "\n",
    "How does this affect your policy? \n",
    "\n",
    "Implement this and justify your answer based on the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Question 2:\n",
    "*Remark: This second question need not be implemented in the notebook, in the same way you\n",
    "wrote programs for the first question. Instead, we expect to see a complete problem formulation,\n",
    "with suitable expressions and graphs, and an explanation of the solution procedure in terms of those\n",
    "expressions. The notebook only provides a means by which to explore the question; this question should be completed by hand.*\n",
    "\n",
    "\n",
    "Now consider two additions to the above problem specification, that are common in realistic versions\n",
    "of this problem:\n",
    "1. The agent must include within the costs, the ides of navigability - describing the fact that\n",
    "some parts of a given map are easier to travel through than others (typically, due to the ‘diffi-\n",
    "culty’ of performing low-level control with respect to the terrain features found at that depth).\n",
    "One way to model this would be to include a distribution of costs over the given terrain map.\n",
    "By re-writing your complete problem specification, explain how you will incorporate this\n",
    "feature in your modelling.\n",
    "2. The agents in the environment, previously viewed as static obstacles defining where you can\n",
    "and can not traverse, could have their own dynamics. For instance, one of the mines could be\n",
    "an active craft with its own motion policy. Assuming that you can observe the current position\n",
    "of this active craft through a noisy channel (e.g., by interpretation of sonar reflections), pose\n",
    "your - now interactive - motion planning problem in Bayesian terms and explain how your\n",
    "solution strategy might need to be altered.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}