Navigation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Navigation\n",
    "\n",
    "---\n",
    "\n",
    "### 1. Start the Environment\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The environment is already saved in the Workspace and can be accessed at the file path provided below.  Please run the next code cell without making any changes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:unityagents:\n",
      "'Academy' started successfully!\n",
      "Unity Academy name: Academy\n",
      "        Number of Brains: 1\n",
      "        Number of External Brains : 1\n",
      "        Lesson number : 0\n",
      "        Reset Parameters :\n",
      "\t\t\n",
      "Unity brain name: BananaBrain\n",
      "        Number of Visual Observations (per agent): 0\n",
      "        Vector Observation space type: continuous\n",
      "        Vector Observation space size (per agent): 37\n",
      "        Number of stacked Vector Observation: 1\n",
      "        Vector Action space type: discrete\n",
      "        Vector Action space size (per agent): 4\n",
      "        Vector Action descriptions: , , , \n"
     ]
    }
   ],
   "source": [
    "from unityagents import UnityEnvironment\n",
    "import numpy as np\n",
    "\n",
    "# please do not modify the line below\n",
    "env = UnityEnvironment(file_name=\"Banana\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Environments contain **_brains_** which are responsible for deciding the actions of their associated agents. Here we check for the first brain available, and set it as the default brain we will be controlling from Python."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the default brain\n",
    "brain_name = env.brain_names[0]\n",
    "brain = env.brains[brain_name]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Examine the State and Action Spaces\n",
    "\n",
    "Run the code cell below to print some information about the environment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of agents: 1\n",
      "Number of actions: 4\n",
      "States look like: [1.         0.         0.         0.         0.84408134 0.\n",
      " 0.         1.         0.         0.0748472  0.         1.\n",
      " 0.         0.         0.25755    1.         0.         0.\n",
      " 0.         0.74177343 0.         1.         0.         0.\n",
      " 0.25854847 0.         0.         1.         0.         0.09355672\n",
      " 0.         1.         0.         0.         0.31969345 0.\n",
      " 0.        ]\n",
      "States have length: 37\n"
     ]
    }
   ],
   "source": [
    "# reset the environment\n",
    "env_info = env.reset(train_mode=True)[brain_name]\n",
    "\n",
    "# number of agents in the environment\n",
    "print('Number of agents:', len(env_info.agents))\n",
    "\n",
    "# number of actions\n",
    "action_size = brain.vector_action_space_size\n",
    "print('Number of actions:', action_size)\n",
    "\n",
    "# examine the state space \n",
    "state = env_info.vector_observations[0]\n",
    "print('States look like:', state)\n",
    "state_size = len(state)\n",
    "print('States have length:', state_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create DQN Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import time\n",
    "from collections import deque\n",
    "from agent import Agent\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "model_weight_name = 'checkpoint.pth'\n",
    "\n",
    "agent = Agent(state_size=37, action_size=4, seed=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. DQN Agent Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 100\tAverage Score: 0.88\n",
      "Episode 200\tAverage Score: 3.78\n",
      "Episode 300\tAverage Score: 7.15\n",
      "Episode 400\tAverage Score: 9.981\n",
      "Episode 500\tAverage Score: 12.49\n",
      "Episode 595\tAverage Score: 13.00\n",
      "Environment solved in 595 episodes!\tAverage Score: 13.00\n",
      "Time Elapse: 623.92\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJztnXmYXFWZ/79vLb2ms3cWyE4SQtgCRJA9CCoIigoj8tNxH0bFXWeI24jjOKIi6qiDoCIyIiiyiIYdwr6GEEJCFiAL2ffuTtJbVd3z++Pec+vcc89dauuq7no/z5Onq+56blXlfc+7HhJCgGEYhqlfEtUeAMMwDFNdWBEwDMPUOawIGIZh6hxWBAzDMHUOKwKGYZg6hxUBwzBMncOKgGEYps5hRcAwDFPnVEwRENFkIlpMRK8S0Uoi+qKz/Uoi2kJEy5x/76rUGBiGYZhoqFKVxUQ0EcBEIcRSImoD8CKA9wL4AIADQoir415r7NixYtq0aRUZJ8MwzFDlxRdf3C2EaI86LlWpAQghtgHY5rzeT0SrABxazLWmTZuGJUuWlHN4DMMwQx4i2hjnuAGJERDRNADHAXjO2fQ5IlpORDcQ0aiBGAPDMAxjpuKKgIiGAbgdwJeEEF0ArgVwGIB5sC2GnwScdxkRLSGiJbt27ar0MBmGYeqWiioCIkrDVgI3CyHuAAAhxA4hRE4IYQH4DYATTecKIa4XQswXQsxvb490cTEMwzBFUsmsIQLwOwCrhBDXKNsnKoe9D8CKSo2BYRiGiaZiwWIApwL4ZwCvENEyZ9s3AFxKRPMACAAbAPxrBcfAMAzDRFDJrKEnAZBh1z2VuifDMAxTOFxZzDAMU+ewImAYZkjTm8nh9hc3g5flDaaSMQKGYZiqc9W9q3Hj0xswtq0RZ87mDEQTbBEwDDOk2bm/FwCwvzdT5ZHULqwIGIYZ0pAxZ4VRYUXAMAxT57AiYBiGqXNYETAMw9Q5rAgYhmHqHFYEDMMwdQ4rAoZh6gKuJwuGFQHDMEMbzh6NhBUBwzBMncOKgGEYps5hRcAwDFPnsCJgGIapc1gRMAxTl6ze3lVTramFEFizfX9V7s2KgGGYukAV+Y+v3YVzf/YE/vzCpqqNR+eGpzbgnT97HEs27B3we7MiYBhmSGPKHl236wAAYNW2roEdTAjLN3cAADbt6x7we7MiYBim7qgdh1AeyxlUgga+8IEVAcMwdQtVQegGYTnximqMiRUBwzBMDSAD14kq6CZWBAzD1B01lCzkItg1xDAMU1lqKVXUhMUWAcMw9cTq7V34xcOvle16//Pwa1i7w5yDb/K516JKsNxBsUXAMEwd8N5fPYWfPLgW2ZxV8rW6+7O45sG1+KdfP2PcXzvh4HDyrqGBvzcrAoZhBpzeTOkKQCJn0pkyKJVqkncNsUXAMEwdUQ4XjfT9FyM+ayh71FUEySqYBKwIGIapGuWI38pL1FJNQDG4MYKh5BoioslEtJiIXiWilUT0RWf7aCJ6kIhec/6OqtQYGIapbURNhm2rgxiirqEsgK8KIeYCeCuAy4loLoCFAB4WQswC8LDznmGYOqQsFoFzjSjxqd6rFlNJh2SwWAixTQix1Hm9H8AqAIcCuBDAH5zD/gDgvZUaA8MwQx8RoQnCJthUQzlFbouJoZo+SkTTABwH4DkA44UQ25xd2wGMH4gxMAxTe5RjYm7FtAhqHVefDSWLQEJEwwDcDuBLQghPz1dhq3LjT4GILiOiJUS0ZNeuXZUeJsMwVcAqgyaoZrO2cpJznqMaXquKKgIiSsNWAjcLIe5wNu8goonO/okAdprOFUJcL4SYL4SY397eXslhMgxTJcoh8/KKoPBza0l3SBdXNQLolcwaIgC/A7BKCHGNsutuAB91Xn8UwN8qNQaGYWqbcgRt4waLax35HFYVLIJUBa99KoB/BvAKES1ztn0DwFUA/kJEnwSwEcAHKjgGhmFqmPIUlJXhIjWA5bqGBv6BKqYIhBBPIlhJn12p+zIMM3goT7A4XoxAdbnUovKQlkA1hsaVxQzDVI9yKoKA/abtUinUkjtJfhTVsAhYETAMU3He3NONp1/f7dtejqyhQtMuV2zpxIotwYvWb+nowWNrwzMVl2zYi9d35tter9jSiVc2d0bee+f+Xjy8aodxn1QAty/dMuDKoJIxAoZhGADAGT9eDADYcNX5nu3lzBqKO7+/4BdPhu5/+zWPobs/5xurysVOy2t5jLxm2DkA8OHfPoe1Ow7gte+fh3TSOw+Xz7Fo+Tace+QEvPvYQ8IfpIywRcAwTNUox8zXKtAiUDGd092fK21AIWzc0w0A6Mv6W2ZbyqaOnkzFxmCCFQHDMFWjrHUEZbhWpWlwrIC+jF/ZlMNNViysCBiGqRrlaTpXfEHZQNOQskVur8EiUD+LgX4UVgQMw1SNclTR5nsNRaSP1kDKaCppj7HXYBGon8VAKzVWBAzDVI+y1hGY94fVFwx0f6K06xoyxAiqqKhYETAMUzXKIfxkkLWQ9QiCj6msNJYxgt4sxwgYhqkBtnb04MTvP4SNew7GPueS657Bn557s2xjMLmGVm/vwsk/eBh7DvTFukb8yuJoQd+juGze+6un3NeXXv8sbnne/9xX37/G8/6t//2wp74AAP747EZcev2zAPIWwcXXPo0/v+C9njdGQPj+oldx5d0rQ8dbLlgRMEydcteyLdi5vw+3PL8p9jnPrd+Lb9z5StnGYJLL1z22Dts6e/HomvK2nxdCoD/ndcnoqkN12Szb1OG+fmbdHnz9Dv9z/3Lx657327t68funNni2feuuFXhm3R4A+RiBJYArbvdeT7cIfvPEetz4tPdalYIVAcPUKXJt3Gou22i6M4XsMxHXpSIA9Bp88yrZMviqwtYc1ovIVNTnGOhgMVcWM0ydImVNbEFaAYVhvCYVdr/YBWUC6DP45r3X8t7TskTBQjlszWGZPmqimllNrAgYpk7JWwTxjs/kytEXSHh8+WY9UJjkjcwaUo4zZeuo5DSLwBLCU/Ebh7BYRToZvK+aioBdQwxTp0h5Fdcbki1UIhqIcy85rrhy0S0oC1y9Pn89n0WgnaIrgpwQBT93mAUR1zV0sC9b0D1LhRUBw9Qp0iKI6xrKZMu3vnDQe0CRzTFvF9c1JER0jMBnEViFP3exMQL1o+jiXkMMwwwEFOKLP9CXxZaOHs82NeNmW2cP9vcWLqx0wd+ftbBhtzl9VaaW7trfh+fW7YFlCby+c79vvJbl7TXUn7Ww3nBNAWGs6FXRg8U5Q6ZRFKoaOKDN7HXXkKXcL6c8V1dv/ryBCOazImCYOsWNERj2ffD6Z3DqVY94tmUUgXjyDx7BeT9/ouB76jLtG3e+ggVXP4rO7rxSySso+++nblqCS65/Fjc8tR7nXPM4fvPEOs818haBfeK371qBs65+FPsO9vvu3Z/V00c1wawNMGcJz3PHIaFEi9/zy/CW1wf6zQK/S1GyhSqiYmBFwDB1SsKNEfhVgVy4RZ2xZp1gsZzVbt7X4zsvCt318sKGfQC8AlEKZ3nk2u12gdZq5+/Sjfn8fkCNEdg88Zpdf3Cw3zsbF/DOuk1kc35rQ98WheoZWrfLa5noz69aKOrQVIUV5c4qB6wIGKZecWMEwYfsV1wUcmaaDMuPjCAoHqEqHN0i0KuPdWHunqoFv32+eiF8gjhqfMW5hoI/H/32ahaTem9V+USlvJYDVgQMU6ckNIFrolMJWkoXSSpRvNgoLGtIeMYnxaspxVPdr6eTSsFsiegUTT1GYBXjGgrQA0IIj8IDvEJe3aVmKkWlvJYDVgQMU6fEqSw2KYJSLIKge3ln4t76Bv0Mn7AOyETSFYYwWAS60eBXMihCEZg/H0v4rRnV7aN+NmrNRlSAuxywImCYOiUsRiDxKgL7uFRJriHzdk/DtYjL57S8ftczpLm69PR/Af+z6o9ucg0VWkgXZhHoika1CNRbq8rHtKxluWFFwDB1iuoyUVFnpiaLoBSClI5pqwgwCUwzfcDvGtILwYQwKALt4qZgccHPHWIR6PfvjREjYIuAYWqcm5/biOfX763ItW9/cTMeX+vvwNmXzeHf//oybnpmQ+C51z/+BlZs6Qy9vh6Ulai57x09+RRMKRDDLIj7V27HPa9sC9xvWQKvbu3yb1cbrjl/pQtIv9+z6/biVqUltLsegXNiLuA82yLQbhxlEZQxRmAZLILeTA43P7cRz67b4xlbxhrYrCHuNcQwJfDNO1cAADZcdX7Zr/3V2142XnvdroP4y5LN+MuSzfjIydOM5/73PasjxxUUI1B78vf051/nFUHwmP/1/14Mva8lzLn1qoDUBbp7jDLOhXe8gg+eOMW5prQIvLEFPZZgcs3ogt9UUFauGIEQfndVX9Zyf0ONSkM6ec8Jw5vQmK78fJ0VAcMMMiJTIGM2D5LJP2F+c6/fWmbxFF/paglhbPXsUQSOQJfHyT1B+fx6i4mgYLG6T+KLERizhgqsIwjYLiAMweKAGIHT1uLBr5yBtqZ0QfcvBnYNMcwgI6pnfqbAJmn65VQBqgouOUstXg3YgtjUgdNkEWTl/QJ8/hJfywnnve4ysoQwZBh5r1UOiyAo2G2KEexVqp/VeIX8DsN6E5UTVgQMM8jIRgimuJWwQemZqrCyTIqgBE0gRH7dXhWvRWCjWwT9Ac3f9BYTUgEYg8W+TCJ/TEB/X6giCNLTlqGOYOf+PmV/fnu+inuQKwIiuoGIdhLRCmXblUS0hYiWOf/eVan7M8xQJdIiiCm4pED3L8aivlYUQdYchC0ESwikDYuz5DzKx9kmFYHzPui5ggrKTMFi3TWjP4o/I6nw7qNBrjsh/Pff0dXrvvZmDVlIJqikmo1CqKS6uRHAuYbtPxVCzHP+3VPB+zPMkCQqRtDvVgCHCxEpeIJcK/br/HbprijFIrBiWARS0em++UhFoMUIdMtICP+z6u91QZ2zhM/VFhUjCUyRFQL6I6iKwNNrKCdKqtcolIopAiHE4wAqk1fHMHVMtEVg74+aTbquIX1WrGzIeSwCGSMoXhPkLGF0d6j3kVaIXjgW5PJyW1BovYZ8s3/4BbH+Uequm5wQ7nPr9wsisJ+S8F9/Z1ef8disZRkVZqWoRozgc0S03HEdjarC/ZkhwsY9BzFt4SLct2J7tYcSyPrd9hgffHWHu627P4tpCxfh14+94Tv+zy+8iWkLF2H3AbOAALwxgs/f8hKmLVxk3B+lCAJdKKqLwrIwbeEi/OSBNa6CUWXZtIWLsHlft+/a0xYuwqNrdvq2CyE8aZISk0WgK7yg5m96+qh6zWkLF+GvL2527h2joCxG1lCUGgzyzH3vH69izY79nm1qjMAzjpzZhVYpBloRXAvgMADzAGwD8JOgA4noMiJaQkRLdu3yF9UwzPLNdsHU35dvrcr946RRvrzJbpn8D2WMHU7v/T88vcF3/C3PbwIAbNzjF64SVVj9/WX/s7s9gSJ6Ncir+LOG8q9lMdP1j69TXC5eSbdqm1e4Sf62zD82S5gDoB6LQHHt6ErJ+BwBX0M50kdzlj/dNSpGErT/zpe2+LZ195uXpOzPWqHrG5ebAVUEQogdQoicEMIC8BsAJ4Yce70QYr4QYn57e/vADZIZfJSSz1gCUb56oHA3ipzFhwkb031V/7nMrklGCJI4MQLZFz+ZINfVEpZuqqKvziWvnU6Fp4+qFkHW45oKcrl4YwRB47Isf9ZOnPRRXfFFuoZi1nGY7i/pz1kldXktlAFVBEQ0UXn7PgArgo5lmFonyldfDG4juJBrm4Km5nbRUYrA/hvWeE0WPCWJAgVgkGVkWoDdEiIyWKzGCDwxigiLQH9aXwYQAH+YITxYbFnR3U51oha/iUvDALqGKlZZTES3AFgAYCwRbQbwHQALiGge7E9/A4B/rdT9GabSlJJGGUQixmIxevGVELYiGDusEUDehRK2iDqQF+Bh6aOy82UiQYHKKRdgWZgUgQhyDRlcQFkthz8qa0g3CUxuIP96x9o4tHvo/YHizPYLsQjCGEjXUMUUgRDiUsPm31Xqfkz9EdWuuNIUYhHEHWpeEQRfW73vsIYU9vdlPRaBdA1FWQT5OgLvdvXesk1ygoKVU05x5aiYXEM5Sxhnup5qZlkQlhNa1pJXAebH62zTrunrNQR/ZbF/9TNtXMKbPpqxrBhZQ+H74zJQxWQAVxYzTNGUa+anIt3CYUVhahplS2MSgNk1lCgyayjnUQT5GEFwWqQwjrm7398+OY5rSKaN6oFaNVisXiNujEAIf0aPP96hWQSWQC6njs2vTHzPUiZLkRUBw1SIJRv2YsPug9EHOty2ZBMeWb3Ds62jux8Pvrqj6BhB2FnSIghrdKYKq9YG26iX2UOLV+/EdqdISVoEz67bgze1LKQXN+7D75/aAABYu2O/m930+s4DuP6xde5xj67Z5Y4rKBZwx9ItEEL4grnbOnux9M19WLQ835b6wVd3+BaVB2xFdu8r2/Dixr14w1nwfdmmDty2ZLN7jLqguxzKAyu3u0rwpTc7sGpbvsW13g57+eYOrNnubYG9ensXfvvEOtd60RWFXxlFh/8tS2D97oO486XNEUeGMyRcQwxTi1z862cAxGsbvbOrF//21+W+4z9+4wt46c0OPPSVM4sag3B96v59eUUQYhEogmnssEas230QD6zcgb5sDh+/8QV3n8xA+uD1z/qe4V9uWuI2PNvR1YcLf/UUNlx1Pq66dzUeWuVVfPJaQXrvidd246nX92DOxDbfvvf/79Oe9//7qL92AgC+dZc/b2RLRw9+eN9q97363JYQ2N7Zi8uctteS837+hPv6/pXe51i8xp+GvmJLF1Zs6cLEEc04/5iJxmUv9aK6qPx+SwicdfWjoccA4e42gC0ChimIUipdw9jb3W/c/opTvxAnWGw6JKw5qPTmqEsY6qiuIZmKaS+M7j0urKBM7Xqp0tlj3p4gCnV59GVzZVnBLAx5/ffOOwSWEB4LoVT6c/bnrVcv5yyvSypricgYQdyPIRUh6FkRMEwN0NmdMW7P98IpThAF+bSBvEUQtiqVKbc+Y/l73SeLyEMPckmFxQgAoKUhVXBztkLJt85IwBLlTRaQCsDXa0gIj3Loz1pF9xrSiWohwYqAYWoANQBrIm67Z50wQSFbKfcYAq2SnJbFYo/F8gQ1gcIXme/PWoHVu8lEcB0BYAvloBYQ5UY+VzktEKlcjcFiPUYQmTUU73cRFQMYspXFDFNO9N4y5SZSERSwAAwp09cwv7CU3b1hriHVIlCWj9QFcaEtjDt7MoGzeqLoIrdCPo9SSDkCsq+MrqG8IvBut4RARvu8o8R8nIpzIHrGzxYBw9QAUYqg0CUMJWGuBSm8Q11Dyn3V1+qyh+q14tLZkwms3k1ScLAYsAVkpV1DEmkRBCmCYnr4y0Iy3SLIWcKzrT9rhTbpAwqxCFgRMEzNIxVBkC+3eNeQ/deYNSSFXCaeRaBaAXqAWW86FxaABuxAcdAzRcUIMjkxYK4hGfsI+oyaimjNEGYReBSvwTXkb2Ud755SYTWnk8b9DYaeTJWCFQFTl7y2Yz9O/9Ej2GNo9/zrx97AF299KdoiCPgfv6WjB2f8aDFe3tSBr/zlZd/+MNeBDBZf9/i6wGPUGaoqpHr6veN5fsNe3LZkk/tePk+Qb72zJxO4DGaCohSBVfGsIYl0DQUpnuYGs2ANY39vFmdd/ShueGq9Z3vO8rviPn/LUs8xYQV5YUhFMKzJnMU/ZJvOMUwlKKaQ87rH12HT3h48vNrfM/+qe1fjb8u2oi8j/e/mGwTNnm99/k28ubfbk9OvEp41FD121SWlCl9TXEHWQQBwn0d3qcyfOsrd31+0RWAZewsVyyEjmjzvZ44b5hkLkH8encZU4Ypg5dYurDcUGuaE3gHVwrPr8uttqUH0dJIwo701NJZy9pxxuOkTJ+LKd8/FiOY0AKCt0awI2DXEMDWAnPEHKwKzIMq5bgbzeWGKK6pRnH7dTEiMQEc+h37cD95/tH1dIQLdR4mAgrIJw22BnckKd52FOIQ95rcvmIszD8+3nr/4hEloVWb5UTGCpnThYk0dz4dOmuK+tpwYgVTQqhUyprUBhPzn+qVzZmNYYypUYX781Ok4Y3Y7Pnbq9EiLwNSuu1KwImDqkjhWhLt4esD+TICgl66BIEVRatfSoP47YQFm9TxdEci4RM4SgbPsJJmD3LLXUcayIl1pKmGprQ1J8ijEVII8fZOky0QWgfnOL8IiUB9N9dnnnBXK5DbVCkxoVlKCCERkaHWdR00JlY/UFqAIanKpSiI6jYg+7rxuJ6LplRsWw0RTjoKisEtIwWlqXwyECHppESjnqPcpdNGZsPuq6+n2RFkEzrX1mXQ+Lz844JtMkDEI2uLM1DPZwhRBmOWju0QSCfIEvuXuIKXVUET+vfqdqDEG2WJCblNdcSnHSlJdfcmINFu1PYX8DFoaBolriIi+A+AKAF93NqUB/LFSg2KYQihmgh2nLYUqcOU9PEsnBkz95GlB2SNhWUNxAo3exVriu4YCLQK3iC3Yx08BwWIpxDI5UTaLQBeAqQR5UkKTrkVg/oCLEaCqS0xXBFlFEaj31NeOSFB0LEWd5cvzg2b+qRosKHsfgPcAOAgAQoitAPwdphhmkEEhM1NV4MpX6mQvKGtICoIgoR5WR6BXB5vwuIbU9NEIRZBzFYFmETgC50Bf8PlBdQQtioDsKsQiCFEEugBMkFcRpCKCxcUIUPUz8bqG7M+4KSUtgvyHIO8jvzPXNRRiEahjk88UVEFci66hfmH/egUAEFFr5YbEDEZylght79zZk8FuQ6pmMazbdcDz/mB/Fjv395Z83c5u7xj1bpcAsGlvvp2zahGoGSdRweKg7ZmchQ17zJ/hG8ozq3EB9VJRMQJ5X11hSLfLyq2dgecmEmaXllQE2ZxARwGKIKzoSxeAfovAfr12x37j+aVaBC0hriHVmpKfm/p9JCPSbNWxyTlI0HhrzjUE4C9EdB2AkUT0LwAegr34PMMAAH7+0FosuPpRYwoeABz/vQcx/78eKvk+j6zegbf95DH8Y/lW1+/+xGu7ceL3Hy752sf+5wOeMaqC3hL2wucLlPbCqr/4rKsfdXvhGwWBaUUtTRZ+f9EqrN7uF25L39yHs3/yGFY7vfT7tQreRsfvHOUakhZKn+ZSkbPzfyzf5jvHPTdgQZbGVBLJBCGTKyxGoBe7qegCMJkgjytJzqpve9Hc77+YmbSqRJsUi8Duv5QPFncr7jOpkOTnKi2XsDCPyTUU1IW05hSBEOJqAH8FcDuAwwH8hxDiF5UcGDO4kLnVO7rMM/O4/Vei2LDbnpEv2bCvQs2n86gzPSGAA5oPXV+YZqMzmzcqAmVTkGvoqdd3G7dv2dcDANi9324R3dWb8dQbyKUfZbD4inPn4JhJI3zXcS2VIprT2UFR//YE2UJaKoK47R1019DtnzkFM8bajoZUkjzfbTJBHmEZdY9UkvDSt9+OcW2NscYCeJWo6hra35v1WATqqmuuIrDywWKi8N962vAcQa6hgYwRRC5MQ0RJAA8JIc4C8GDlh8QMRmTwtdI/XZlqV4g/ulg8MQLhb0utZw3JYqwopRe0O0jASZeLFFZdPRmMbm103ViNqST2I+vOao+bMhIPvLo98Hl0BRa1pGVjKgErwCJIJuxZbr+jCEa3NmDX/mgXYEpz9ZwwdZQrbPUZfTJB7jY7Myeqa2cCo1obPDP7KNRMKjWLp7Mng4wSIwhTBHmLwG+xyeub0keDvveaihEIIXIALCLyTzEYRiMs+FoO2prsasyu3vJVsQaR0VxDuutDbzonUzmNySwe11A+LVUlKKVSKj0pTDq6MxjT2uDud11Djp87qGW0TGvUBVWURdCQTCAnhNGSSSYI6VTCtgi6Mxjd0mC4gh9TOq07Q04lPPsTRK4ATRBFFt0VI0DVuElzQ/78zp5+zSIwuIYUi8AUI/AEuk2uoQRhmKG6eCBdQ3GXqjwA4BUiehBO5hAACCG+UJFRMYOOMq3XHYmsGt3fmymtMKuAgjLArAj0tsv5ltBRFoF5f9DMsFOzCDp7Mpg+Np+v0Zj2xggSRMbHywZZBFEzbGkRGBScFNL7e7Poz1kY1ZoOvZZEHYH8OOQ49I8hlSBXKCYoun4krVgPcenNmmMEnT0ZZC3h/u5Ui0CON68IZNaQ99qqBRMUIxjWmHLXTc4/Rw25hhzucP4xTCgVNghcAdLVmy2L8olTUAbY7hyfItAtAreVcXGKIMhFI11Svdkc+rMWejI5jB6Wn3lL4SIVQSpBRs0sg5p6wVNci8CUDmvHCBKum2pMazy/vOkjkIrQEl5FkVBiBBTDIijUt666buR7iWzEl0okkE4SuvvCXEO2q8xXUKYMJ21IH00lyG4z0eU9reYsAiHEH4ioAcBsZ9MaIUTlnbTMoGGADAL3P9mBvhItghh4etML+Hrp6K4hN0YQZRE4l9XlWZD86nDWEe7L5DNzVBdMozODlXn1QZkrMkisWwRRwdd0yp7lBrmGGlIJN5A9ujWea8hU0JfUZtieeyTz1kJUU07XIog1EmBYYwp92fxazap7U1oESccqOai6huR4ZWUxzDECdRymzzqZIGObiZpTBES0AMAfAGyA/VyTieijQojHKzc0plpkchauvn8NPrtgJka0xDP1JfrP/KFXd/gETxDrdh3AvSu24/KzZhr33/PKNjzxmp1Zsz8iRrBm+37c8OR6TBvbipaGJI6fMgpHK5k0d7y0BQBw74rtGNmSxtlHjPddQ08f/dXi1737Da6hfyzfiodX7fBd67l1dkvof5o/2RUU2zp78dsn1mF/bxa92VxwsNhRQF29GVxxu91NdHRUjMAgaIMsgqi4TkNSBov9+6RraI2T0x9XERiv5cg9yxK+31HeNRRtERTqUtGDyur1d3T1uddMJxOeJUTl9/W12152zoPTayj481Vfy+8o7biGSn2OUojrGvoJgHcIIdYAABHNBnALgBMqNTCmetzzyjZc9/g6dPZkcNVFx5R0rU/dtCT2sR+47lnsPtCHj5w81Q0Kq3z25nwf+N5MLtQiuOAXT/g1YlE7AAAgAElEQVRm7BuuOt933EOrduChVTuM+9TK4f6chS0dPd79etZQ1sLn/vSScTxbOnrwb39d7iiC/Pb/WrTKfX3UocON5+7rtmerNz69wVUKsoUxoLqGFIvA4M9300cLtKTSyQSyIVlDat/8C46ZiMdf24XO7gzWhRQYmvjue47Cd/++EsdPHYUpY1pw83Nv5sfgKLuk44cP45TDxgKIn7igfq9nzG43tgJvSidt15BBEazY0uXczwkWWwLtbY1u9lSQwSW/o2SC8J13H4lv37UCz6zb4+6vxRYTaakEAEAIsRZ2vyFmCCJnwoWsCRvWNiEuMjsmTitmIHwlqGKXkfRcQync6jd8FnphV9yFWYI+q6DeRXsPOhaBEqNobczPYqVAkumstkXgJyh9NIqGVMIuqDPO4vNVv+ccMR6zxrfhzs+eii+cPQsAcOG8Q4zXNH0Eh09ow5/+5a1oSicxaVQLLjtjhrvPdZOQV7B+YP4kzzV+celxOGvOOHlowfzwoqPd319bU8qtMh7RnEYyQd7KYk3Ck1JQpgaFgxSSVKypBGHmuGG45bK3evYnB3BhmrgWwRIi+i3yjeY+BCD+VI8ZVMjfbTHCvZRgsWzoFfeulY5LqA3GTEJedw3FXaoxSA4HKZIOxyJQz2tWct1lsFfGKJJExu9OKoKw7pgm0o5ryBgjUPoAxSlMy1PgGOS1he2Ll+jLPHomEUX8FlOJBLKG9tbDm9NIJRKeDq8+RYB8QZlqPQUNQ34NQZ9bYZ9nacRVBJ8BcDkAmS76BID/rciImKoj/zMV8l+1nEI5bhC40sHi/qy9IIklzBaGLri7Q5q2qQSNO8iKMc3gPQu1OC6EjGIRmCjaInCyhsyuobwiKGTR+EK/OukaylrCYxE0ay2cS5Wd6toHhPw4RzankUoSOnvysSndck0QuXUEHkUQMDuSijUZEBSOaxmXg7iKIAXg50KIawC32jg0T4yIbgBwAYCdQoijnG2jAfwZwDTYgecPCCH2FTVypmLI318h8iL/uy/9xyvieqQKFCaFWjj9OQvppJ1aaJqt666hOBW1QT177OvFd8WpM2Hpo1cVQViL64ItglQClmVOi5XCD4iuUFYpVIVL11DOEh7BqlsEJRoESCYJCUMegnQNqcFifcZOlG9Drf5cAmMEimvIOJYBtAjiOqEeBtCsvG+G3XgujBsBnKttWwjgYSHELOeaC2Pen6kChQjOwWARFNruqD9rub5ek9tHVw5xuqv2ZYOD3FELy6gkE+RmC7kWQVZNHw12DRVlEYRkDRXjGir0u5Ppo1ll2UjAWwUMlF7Znk4kPK4nyYiWNNIJb72BLqjzbai9/3eChhTlGqpFRdAkhHD74DqvW8JOcFJL92qbL4Sdhgrn73tj3p8ZNJSuEuIICYI5IBpGMTEP6ZLIGGbrRSmCjBUY5NYrS8NIKIpACgw3RhAQLO7PWpHZViYaUhTSYiJ//0q6MmRBmSW89wmNERRBMkHGGfzI5ga/YNbfyoIyrfjOpFgAxSIIyA6qRUVwkIiOl2+IaD6AnpDjgxgvhJC9brcD8CdvM1WnmBiB9EVcdO0zmLZwkduJsxjiTFgFgl0swecEM23hIuP2tOt/j44RxHEN9YZYBHLGrgs3wD9rTBC5+e9yn1REQT3xv3P3Ssz59n2BHWKDCK0jSJgtgjFO5fPkUeb5YiFfHcFbXKUm0+g1AOqnVIx1kErk01OJCEcfateejHBiBCp6F1dSYwRWtEUgP4Og7KBaDBZ/CcBtRLTVeT8RwCWl3FgIIYgo8OdARJcBuAwApkyZUsqtmAIpJWtIsnxzJ6aOKW79orj3LdTVU8zjSAFkjBFoguBgf7RrpzdjRY7jka+diT0H+vH3l7fiusfXAbAFnmoxJBN5RaDPHJNJCtV6hSqCdDLhC4C691JcQ2qM4PRZ7fjtR+bjzMPb8UutEA8o/LelFld5YgQNmkUQY2p74vTR+NRp0zFz3DD05yy0NqRw+o8Wu8+QUH7/v/nIfKzc1onmhqRPMOu/CbegzBIQwhbkdnA7IFgsC8oCBH4hMZdSCf3YiOgtRDRBCPECgDmwA70ZAPcBWF/E/XYQ0UTn2hMB7Aw6UAhxvRBivhBifnt7exG3YopFmrKF/F/VDy1lNhNbwBccIyhcE4TGCAoI7krCYgQAcMiIJkwc0YyjDh2ByaPzs2l9NpqkvGtIn/kmA5rO5cdgFeR2SKeCYwTJAIsAAM6ZOz6wTUKh34SpWRvgXU0M0Kp4Q671jiMnYEb7MMyZMNzzOevXGNGSdgvUUpqW0X8TbosJJyFAfi6BMQKloMzEQFoEUfrzOgCyCcfJAL4B4FcA9gG4voj73Q3go87rjwL4WxHXYCpMfkZUwjVKUgSVsQiKQS76YsroiVtAptKbsUIre7sD8tR1gZpI5N0i+kdtZw0F36MvY0X29FdpSCbshWkisoYK8mkXmj7qUQT57WHtIYKvFX6M3OtTsNrz6QWACafYzRK2my8VpQgiYgS1lD6aFELIgO8lAK4XQtwO4HYiWhZ2IhHdAmABgLFEtBnAdwBcBXvZy08C2AjgA6UMnqkM+fTRArKGtEMLETS+a8W+Z4ExgiIUh56jr1Jo9g1g970PO02tRVDlju4+ULOGdIERtVxif86xCGImKaWT5Et7lQIvmSDbFYUC6whiH2mT8izokn8dHiMwX6shFT7/DTpPF9h6QSGRPQHKCQGI/OcRFCyWH6duabj3q6EYQZKIUkKILICz4fjs45wrhLg0YNfZBYyPqQpFBIs1Ssl4iJPnTvALOyFEaIDQ1IgtCjkTNbV/KCTvX9KbtUIVmOpuUJ8lrQmvpBIs1p/Ydg2FWAQhDe5MSOGmft5EBAhhB4uLsAgKTx/1L/oOhGcNBQngoDWCo/DHCLzPYIl8ryFAzaYyXy+qjmAgYwRRiuAWAI8R0W7YWUJPAAARzQTQWeGxMVUiHywu/hqlKII49xUGUSdEeIuLYp5HCiB9wXegWNdQLnZBlyrUfFlDCXIXS9GVXyKgoEzSlyksRiCFm9AsghxkD/7Kp4+qijDMIlAfK9AiiFAEcX8n+vefcwLDlrB/mzIbKLLXUMB4BjJ9NGpW/30iehh2ltADIv9LSAD4fKUHx1SH/M+vkIIy77FBP+JlmzqwvbMXZ85uR3NDEls7enDPK9vcdEMg/mxRn1lbQiARUk9qWsc3CmkRPPiqv7X0zhjpojp9WSt2bEOVD74YAREaU0n3tU6oIigwWJx0LQI9N14U3WsolrJ3DhLQYwTBCjJOOXHcceofq/696VZi1rKQUNxyMhYRdDd5dpBeqqVgMYQQzwoh7hRCqEtUrhVCLA07jxm8yBlMKRZB0Gzsvb96Cp/+44u48u6VAIAbnlyP/1q0Cl/+88vuMcVWFkcJWPUecZH+5MfX7ir4XBN9EQVdnzptuvvaI/BCsoZM8iI0WJzNhc7e50xoQzpJ+JfT7bHIlhVZQ2686hoqrMVE/lpql1GV846eCAA4c3Y7Jo5oAuBvE62vfxArWGyIEbz/uEPd1t6tztoAn9PWxdA/0fc4nVWPmzISgK0Y1NhYZNaQ8I95ipLBVEvBYqYOkT+/QvSAT+5EnLy5oxuAua1C3Bmzfs9iYgBR6HnqUbS3NeKFb56DE7//kNFikDnmOqb1EDwxAkPWkLtYiyIZf3bJPADhH39f1kJTKvi5fnzxse4iPt88f667II8aE5FDs9cHsF8XaxF8411HGI85fsooz+fy2vfPQ5IIL23KtydraUhiw1Xn4/jvPYi9B/tDhed33j0X3/37q0bX0DXO5wbYyt/0fejK9a0zRmPDVefjir8ux0tvdvga4uWzhsyxAnk9dcyP//tZbnFjTVkETP0hf+4F9RrSDo1a/ET++E2z42ILyirRjNRU5RuG9NsHuV6ylnntXxPq7DKtZZYkE4R0yj/jjBPf6Y9wDemyVLo41D478vtTr1PJrCF7HAkkEt6FafLWqxSq8O2TSEVW7Mpf/s/U+QzcPkjCo5TzWUM2enaQZVAEKjVTUMbUJ5bim42LfmzcWb0pGye2RaDdtRJtqfWCpSjC/PaAvXhM3HF6ZpcG15C6fKO7PRGsYCVRMQK/InAC5hk1tdWvhAprQ138dxU26w/zpsjgbrFrAfs/U2/WTy7n/Vyl4DcpTSB47epqwIqA8SF/76UUbEVlxsjZmikXP3bTOe2wShSYFWsRBLU6yAa4hkyEu4bIdXGo8kVaEWG3yFkiVGjrgtZVBKpryHBsKbUjhWCMiTh/w9KHZbpnsemjQb83KfB115AeI9BdPVEWwUDCioDx4WZrFOQaKmx2Lv9PmFo3xFEEAv5umOVYLlNHT0+MQloEQUIxZ4nY6aPeymK/RSCtBFWQSHdC1GdRmCLwu4bkIcVbBLEPjRyfej1vHYEXaRE0FOsagv57s//K70Gmj7r3126T1O4rz68BPcCKgPFTjpm1aRETFfnbzxoUQVwhod9Cvi+nQijUNeRaBEGuoYCePSY8lcUGi8Dk4og7Kw87Lsg15D1G+r8HLkaQv7fheoYYgY60Pot3DZnv6a4ZrVla8v+A/MsWAVMTfPLGF/C3ZVsijyuHrz1K2C1eswuX/2mpsb2zEMDnb3kJtz7/JgDg63csx68fe8NzTG/GwjUPrtXOE/jbsi341B/Kt5x2oVlD0iIIevxsLn77bG/6qF5H4F21SyIF0bDG8ITAcIvA+94kOBOKReDOxgtQBFHjCyM0RoDgGbkUxC3F3jtg4iGfJZUgz9ikBSJPG+6kp0pkmmpQr6GBhNNH64iHV+/Ew6t34sJ5h4YeJ3+4pSiEOOcuWr4Np88aazz37y9vxd9f3ooPnjgFtzy/KeY9gS/eGtoCq2AKdw2Fz61ylhV7Nkwei0BzDSldP1VFIIXxTZ84CQ+8uh0/e+g144I3yQTh9x9/i/tl33rZW/HB65917mt2DXnH5k+JDLIy7vzsKdiw56Bbx/Gt84/AKYeNxbv+5wnj8VEYXUPuuILPu/ysmUgmCJfMn1zUffXftHQVffK06ejN5PDRU6bhthc3u/vl9zJ2WAM+MH8Ozjtqouf8az98PO56aStmjC2uXXs5YYugTihkndp8jCD+9f2BtHgnm/r1FKuAyuESOqzd+5+yUNdQlNshE7JmsY4nRpDQLQJyZ7hqwF0K4yljWvCp02cECsZEgnDW4eNw1pxxAIC3zhjjubZKmEUAkDFeoHLclFF433GT3PefOn0Gpo0NXeAwFKPhYYoRaMe1Nqbw1XccHtl0Lgj5KUvFKL/GpnQSX33H4WhKJz3KULV2LzvjMF+764kjmvGZBYeVvLxmOWBFUCf0FdAgzSpCEehExQgk5qyh4u5ZjtiGLvQKzRpKuULCPJigvv4mVKEmawYkwRYBfMcZxxmWPqq9Nys3f/poIQQ1hIt1bsz4Rin3MKHHBExfsfqx6t1JaxlWBHVCXzb+wujy91uKa6jYxl32ucVbBDJYWyylKgKZ0lmOGIEq1PRipKTHIvAXegW9V88PIo5FEHX9KEqZBIelj4ZZBKUi9a387EzfoxonMdXI1CqsCOqE3kz82YnQ/sY7x3t0zjIvdq5jXvClSEUAoLWhtLCXPoMuNOdcCs2gR89a0UtVumOJ6D4qu1uagsXucQHCsJjKYuOxgXvCKU0R+APicbKGSkVXNqavUf3OpLVbiYr3csOKoE6QFkFUC15AmemUGCOI4x4yrwVcnEltCVFwlo9OlCCN6v8SlQGSLaCOQJ1dmhqluRaBojj1GXqQG6WYyuI4x8alHG6b4U2KInDHE1xHUDJS2YTUaqiGm/z9DwI9wIqgXpAWQZw+K/n2v96fcE9/Di9s2OvO4nd09aLXaTvgK74X8Xz2Wzv8C6n3GRrRxcESpVsEvq7GpO8P//zyrqGAGEHOHyMIDOiqWUMGwe3GCER5LYI4rqGoAHEUpVgE3f3276OtKe3bV8m4q+saCo0R+NNHBwOsCOoEKbDjZEy4BoH2Q//l4tfwT79+Bre+YOf3n/TfD+Pjv3/BeA29f30Qpu6jhQS2VSyreItgwnC7zbEevPbNqLW3epaRFJpBM96M5e819C4trVDiCRabiscMwWLd9x+kuMIsG/2chlTwsYVkvJwwdVT+vNhn+RnRYiuAdx41wd1mqiwut1Y4Y7ad6nyh03560qhm3zGqgj1x+uiy3r+ScB1BndCXjd9wywrwDO3rzgAA9vfm89KfWbfHPtbQYqLYYHMxS0BK4ri+dP7x+dNw25JN+MMzG33uLFWwfOv8I/Cj+9d49t/26VNw/PcedN/Lzzco7TRn2S6z1oYk/u9TJ2HC8CaMHdZoPDasoAzwVrS652iHyWukEuQ5Ti9u8p7jfa8GqmeMbcW63e7SJPbyNAETB52bP3WSW9NQSsrkoSOb8dTCt2Gio7yBvAVWyRjBV95+OD781qmYMLwJnzp9Bg4d6VcE8vOeNqYFnzh1Op54bXflBlRGWBHUCdIiiKcIZJBL761iv4/j47aseDECE0VbBEIgU0TK3rSxra6g1QPVqmBpb2v0zWRbG70CX8YI2prM/7WylkDWEmhuSOH4KaOMx+TvrVoEfgmX73oZ4hpKyPMTyFp562tEiCLQH1KNT+i/n0IEelM66RbolSqwdSFstAjKTDJBmDii2Xh/ibz/pFEt+RjPIAgSsGuoTpDCNZ5ryBzkyqeVGpSEfmzMGIGJQlJd9XsW45dNKX17cpa3sZrq4jEJGd0VI4VzUAuFbM6CZYlYi454K4tDXEMi2jWk3y9MEQQ1nQP8KZMJKi5eMBBFVJW/gx+142g17l8srAjqhLxFECNY7PzVBbkUODkR3Uo5bvqoCdU1VMg1LCGKyt1OJsjtSKme39aYMi76op+rIoVoa4AiyDkWQZwGbd7uo/FiBHq/HzkefXshikB1t+mfbrmLtorFTe0cwMVcTOR7MFFFrZNyw4qgTijINZRv42ncLoRhla0i00dNqIrAVHkchBDFpZ6mEuS6htTz25rS3hbPhv/Y+sxWCqIw15AlROB6BZ5rhdQRAHlFoFpBukUg3+qnhykC/U5qfEK3CGpG1rmuofymaoxN/bxr5rOJASuCOqEcweKM0lY3SsjbweKChwnAGyPoLSCVVAhRlGuIKO8ayngUgdciiDPZlMeYXEPJBCGbsy0CvVI47FqAeaZr6j4aVFmsK6yRLcW5hvQfRa3JOk/30SrcX41VyI+xEmtplxsOFg8BsjkLP31oLeZNHoWungw27+vBB94yyQ1sAeHpo/e8sg0JshcF//I5sz29hl7YsBcb93Tj4hMmuf7zlVu78CWly+eKLZ2+n/odS7fg2XV7i3oeNUbwg3tXxz7v1hc2IZMt7j9d2uQaakppLQsocpaXr3r1C9qWdBJLNu5Fb8bypZ0ar5WIZxGE9hpyu4TGdw2Rdg214Z1buCX/1si015Q1VI2xya+CoFQg174eYEUwFHho1Q78arG3X//iNTtx1+Wnuu+ly8MkUD5781L39eiWhnzBDARuemYjlm7ch4tPmOQKycfW7vKcf8EvnsR0rZXu6u37sXr7/qKeR7UI/vTcm7HP+92T6zF2WAMmjWrGEROH48FXd8Q+933HHYq7X96Kc4+agB/dZ6eIfvc9R3kFS4zryOMvPWkyFq/ZiWWbOtx9jekk9h+w0yfjWQSqEvLvnzd5JI6fMhLfvmAuLvjFkwD8MYsfXHQ0rrpnNb51wRG48u6VOPrQEVi2uROzxrXFui9gK6TzjpqA9x8/CSNb0vjx/WvwjXcdge/+fSXmTR6JQ0Y2YeXWLpx75ISAK5o554jx+Kf5k6IPjEF+ta/82K9895H4j7tXYGb7MBx16Iiy3Cd6HMIdR7Fq6DMLDkNriRXyhcKKYAhg8oZ09WY87+VMOSpI2ZPJuTnwQgAd3f2uuyTMX19sYHhcWyN27u/zbCuljqA/a+GCYw7Bl86ZhXn/+WD0CQ5jhjXi7s+dhrtf3goAOP+YiZh7yHC8vvOAe0yCKDI4Kmfx49qacNflp2LawkXuvuaGhO+40Gsph5iC4E3pJO747KmebXqM4Pgpo/CXT58MAL5jgzCN7NoPn+C+/su/2te707ne1DGt+PvnT4t1bZXffnR+wecEke8DlN929KQR7hgHCkuJVRRrkVxx7pwyjigeHCMYAph+b7rgiOs7t5Q2yUIAXT0Z1/UQqgjiDTUWxdYRAHYdQDpJRWePuGe5M8z8vkIDvDot6fy8K076qHqtuC2Ny5E1M5iyXXSq7arKWwQcLGYGGNPvTRf8stAqqtpXbQ0hAHT0ZPIWQQV6p5j+sxRbRwDYz51OJmKv3Rs1Hj1GEEWYHG5SzP1YFoFyTNyOrMU+t8pgEmCSgeg+Gmsczl/1dzMIQgTVcQ0R0QYA+wHkAGSFEOWzEesQk4DSBYd0DUVNLO1iMfu1EAKdcS0CZRdRaQGyvgJaZutkLWErghIlgjH4GOO8MGXRrKyVEM8iyL+Oq4Tr1SIwdR+tBuqC9IPpU6xmjOAsIcTgaMRR45j+7/drs2o5q4+yCCylEMwSAl09GWOaoo6aIpcg8tcZFEApriHAzowqesEUeDM99MriqMuGzcjVRW7izNy9rqGYFkEZFMEg1AMu1R67+zUNss+QXUNDgFgWQUxFkFViBPt7s7BEXgiFzUpVS6MQ94Rp3lRKsBiwZ9vFCkQ399sUI4gjvEP+R6ljipNb7m1pPHCuoUFpEQxAr6F443BiBIZttUy1FIEA8AARvUhEl1VpDBVl8eqdmLZwEd7c0x157LfuesWTXRLF2695DO/736dw4S+fxDnXPGa0CNTg4q8Wv45bX9gEID9jWbR8G6YtXISdXd71ANZs349rHlwLANh7sB9Avl1E2Kx0S0eP+7rU/4vPbyiu/kCSSiaK9hWPamkAAEwebddgqK4WIuDwCcFplwAwptXcSRSw00clcWSD+gxjhjVEn4DyzIir7WcvhWqPXf5+poxuqbp1UgjVcg2dJoTYQkTjADxIRKuFEI+rBzgK4jIAmDJlSjXGWBJ3vLQFAPDSpn2YMqYl9Ng/Phs/Vx4AXlNSGgHzf351BvmLR15zX0v3zo1PrwcAT0thAHjy9by3TnXRxKkmlkTNxo+dNALHTh6Jm57Z6Bn7e+cdgruWbY11jzCa0gkQEf70Lydh4ohmfO22l/Hixn2eYyaPbsamvT2+c08+bAyu/+cTsODwcQD8LQt+/7G34KU3O/DxG/PrMPzt8lPR1pTCazsP4PRZYz3XW/y1BTjr6kcBAFe8cw6Wb+7Apr09sQKI6uz24uMnYXhTCjPHtYVaTOVxDQ0iCaZRbYvg9Fljcd0/n4C3zRmH5ZvtGpLatweqZBEIIbY4f3cCuBPAiYZjrhdCzBdCzG9vbx/oIZbMQJqDUWEp1V0gxyVXeQrqm6+TteK3b4j6zzh1TKuxBfPps8rzPTel7Gc65bCxmD62FfOn+u913OTgFtDvOHKCW4GtxwhGtjTg1JleYX/s5JGY0T4M7zxygk+IqoV2U8a04OOnTI/9HKo1kkgQzj1qImaOG4a5hwwPPKccriGmeIgI7zxyQqxWLrXEgI+WiFqJqE2+BvAOACsGehz1hCqYc5oiiEu2AIsgalKaTibyrYuV7VHr/calMe39WXcpC+lI4rTjBrR+P267huLHJp8xzkShmPtUu/tmtam2RTBYqYZraDyAO52ZUwrAn4QQ91VhHBWlGPNaCFHUeVECWhUOMnRw0FkpKu4sP5cLjxEE3c9EQ8oczC2HWwPIWwSS/VqVtT2GeIrAsxg6+bcVSr59RzQs1AqntvRgTQ0mlAFXBEKIdQCOHej7DgYsARQzKY5K1VQFrMwa6nEsgrjZKBnLit3rP8o9kU6a0zvj5NbHQbcI5PKIKnGXtPRaBP5thSIbuMXxHJZLMdYTtag8B0HSEKeP1hLFtFAGot0M6n8OqQi6M1IRxLQIrPgWQdSMOZUwK4JknB4OMWhK6xaBXxE0FmURkG9boRQi3GtQptU8tfSZ1dJYomBFUCZe2LAXO7RUTBUhBO55ZZvrxnlg5XZfK4WdXX14+g1vjd2LG/dha0cP+rI53Ldiu/HaQbL88bW7sGj5Nk8DurU7DuBvy7a443jpzQ7zyRqb93Vj94G+6AMBRE220ynyrOQkKZdFoLuGDpQpRlCO0bkxglj3HkSSpEYYzBlP1YQVQZn4p18/g3f+7PHA/Xe/vBWfvXkpbnhyPZ5fvxeX/d+LuErrtf/+a5/G//vNc54Z/kXXPo3Tf7QYP75vDT79xxfx3Lo9vmsHuYY+csPzuPxPS33phl9U1hKQNQNRXHTtM7GOA/KZNmfONmcBNQS0gCiXK0R3DX3klKkAgOHKqmENyQQmj27GidNHh15LFSzFCuaz54zD6FY7vzzluoYKKyiL4qMnTy1qbEOFz79tZrWHEMgg8AyxIignHd3+oKS+b8Oeg9jXbRdqbdrrLTaTM27db5+zBDY6x3b0+O9hxXTZDDQnTB2FH198jG97KTGCH110DG76hJ1t3NaYwufO8gsA3SL40ElTseGq89HWlF+MpSGVwBP//ja3pXIQqnIqVhH87mNvwdJvv913vSgK0YvfvfAobLjq/EKHNmT46jsOr7nnH0y2CSuCAaLJmaX2Za3IxfR6Dd035ZGmiWSxawNXCtk+IZkwL+CdCmgTHSv1kfICua0pZUw51S0C91Tl0LiuIVU5lcPrIK8XJ4DIbo4hwiCIFrMiKAMmM1/fJgVPbyanmIrmH4hpnV55pCm4W2yQOQ5xC85U5KMnE2TsvdOgtIBQZV2c2XKC8vGF4c1pY4aSbhFI1EMbA44x3c90frHkYwS1LxyY+oEVQRmIMyGXsjpOZ82wNswmoV9qt84w4va4UZEfRzJgRa8g11AcRaAe0daUQrIAi0ClGIugHMHbVAHpo2aW+cAAABfnSURBVAwzULAiKANxVo+Ss/zeTC7SNRS2MEuPwVowWRDlYnRIE7UgVIvAJDtTyQCXUazFWoCDTg1EW1Pas7C6JCg1VFVK6ZgFG4kyK4JkAa4hZnAjXXuD4atmRVAGwnz0Mm9fztrjLLrS6xxjuq4pJ76iFkFr4RYBImIE6RJcQwTCgT47YN7WlDKeE+RbL8Y1pFKOpKZC0keZwc1givDw4vVlQM3yWbGlE5f+5llXYF9172ps7cjXF8RZhrEvm8MXbnnJXUgdAB58dQcAc058KUs7RlGcIrBJBFgEQemjqRgFZURAf0Y4Y2ssrEBLed1cROyjnMFiZugjf5t6gWMtwhZBGVBn7t+8a4Vn1r6jqw8/vn8N+hz3TZyVu7r7cx4loGJql9Abc2nHKaPD22G3NXnnBZ87aybeceSEWNcGgBOnjcZtnz7ZnZGnAiyCM2e356t0FfEcyyIgwoXHHYLPLDgMX3nH7AIrde1jT581FqdrHUQLOR8Avv++o/CPz59W8DUKqSNgBjdHHjIcXz5nNn5x6XHVHkokrAjKgBoj6On3C2og777J5qLzRcIqeLsMDdTixAhOOWwMZo0bFnqMniH0tXcejrFKsPiM2e2hQdaLTjgUb5k2GmlHOCeVDB/JuUdOwKjWhqILyhJku3WuOHcOhjWmYvv6gbxF8OW3zy6qS6eq1D500lQcdeiIgq/BMYL6gYjwxXNmYfzwpmoPJRJWBGVAbcZ2sM8slKWw7s9ZkXn/O7qCFYHJNRTHIujuz0VaI60Nfk+h6q5pSFKoc1v2Cko5PSbsYLFZ4BpjBAWu46veMxbk+VMw5XDqcGkAU4uwIigDqmDvDrAIpLDOxmjnHNazyBwsjrYIuvuzkQqopdHvy1QLttLJROiax7LHkJylJxP+5FEpCE0uozhCUj+kEJ+7PLLYQq1yZA25ayJzuJipIVgRlIGsRxGYhfIBpf9/LiLddGeYRVBkjOBgXy5aEaRNFoHSFC6ZCBVfUlCmFYtAF55hiiCOu0YX4ro7qSmkhkCeW2y8thyzeaka2TXE1BKsCMpAVinyCkrl3LnfnuVncpaSZWSWBvJYEyZFEMci6MlEKwJTJo0qaNNJCg1yplzXUN4iCPLcGNM+Q0fnHKMdpFsEwxqjE+GilvYMohyrf+UtAoapHepWEeQsga0d+cXLhRDYvK878PjN+7rd5m7bOnvQ3Z/Fixv3oS+bi9Wn/4UN9uLp/dl8jKC7P2e8Z1iMwLTa1pYO/yLsOgf7sqFuHQBoNbiG1LVXoxZzkbulQjDFCKQQdmMEyr44rhf9GF04t4YoAlNcohDKEiMowzUYptzUrSL40f2rccpVj2Cn44//8wubcNoPF2Ppm/t8x76x6wBO++Fi/PrxN9DR3Y+Tf/AI5v7H/bjo2qdx09MbC2r6llUWeHn6jT047YeLfcds7wy2CLp6/BbBul0HI+979hHjol1DpmCxFiMIu4IM3LoxAgqeexubzimbDhlhzrTQz5o2ptXz/p0h6a7FWgKScsQIRrbYWVhnzDK36K4mY4cVXkXODA3qtqDssTW7AAD7ujMYN7wJz63fCwBYv+sgjp8yynPsNqcg7MnXduOCow/x7uvsjd307dhJI/Dqti7kIo7vz1lIJsgjuN877xAs39yJdbttoX/7Z05BKkG44vblWL19f+j1bv/MKTjykOG45DrvmgJEXl+1qcGcWoGbSlKobzsfLM5bBL7DNVmqrwD2/DfOdt1rp//IVpIfO2Uabnx6AwD4XE2HT2jDE/9+FgD7WQ4d1Rw4vrD4RBzKUQvW3taIpxa+DePbak/oPvpvC9x6F6a+qFuLQLpJ5H9u973hE1GP0bM9OnsysS2C2ePbkMmJWOsEj2lt8FT1zmgf5i5uAgAzxrbi2MkjcVLEwioAMHl0M5rSSV/6qJ4uaooRqH17ol1D3vRREJCJaH/hdQ0B44Y3YfLoFo9SmjomXwhnyviZPLoFk0e3YMqYlsqu81umSx86sjn/GdUQwxpTGMNWQV1Se7/GAULKbilXpDAPy2axLH8wuLMnE3st37HOLDBOAdiI5rTHlz28KeUpVR/enHaPi0IKcN0Q0WMCpqwhVRGkA4SXlL2yDkAWlGVzAv3aTeUjmSwL9bMPel0OWVxs6iYvHckMVepYEQjje9OMUgqAnBC+pnGdPf2xLQIptE0Lz5iOVWe/LY0pVyirzdaGx1AEaVcR5MeeTJCvB4qpajilNIgzLQIjr6X+lcdlc5a7TKZpjWJ7Q/6lR/h7un6ajykUtxtkkSk7rAiYoUrdKgI5KZQTVinMTdWtcl/OEj4h3tmTiR0jkDPzoFoDlRHNaY8AbG3IWwSqFRDHIpCCWVVYTSl/47egdg3yuGCLQFcE9nEZS7iKIOhcFVIO8SwaX6bFYUoV49wvjhmq1K0i0Ff8krLcNFmUx1hC+Nw6hcQIpKCN5RpqSXuyXFoaku6CK6rwl1koofd1fPfqMJvSSV8OfpSgD4oR5BWBvF/eIpCfnTw3TJaqM+4gX39pFoH9t1iLoNSsI4apVYZ01tBtSzbh2XV78fm3zcS0sd40Q+kKkkJcvv/BvauQIDu18+QZY7ByaxdmOs3asjmBnzyw1nOdHV19uO6xdbHGIwXtLc9vijxWjxG0NCTdDJ5CLQLpZlEVVmMq4Wv7HKQI8hZBlGvIGyzO5oQbU2lIJQBPeYRfGge5gFSLqxRRXGp7B6rbaRMz1BnSP+03dh3E7Us3476V23375Kwwa3kVwqa9Pfj0H5fipmc24jM3L8UvF7+OK/++EgCwdsd+LNvU4bvW8xv2ho5j0qhmfPmc2bHcI588bToOa2/FqYeNdYVhKkE4ZtJIt33CyJb4iuCEqflU2B9edAyOnWR3zGxKJ30+f1XQf+yUae5r+VmpMYWfXnKs+1oKcPn38rNmYs6ENrx97nhcOO9QHD6+DX/4xIk4fHwbvnD2TADAlNGtmDd5JH7wvqOV65gDxP1KgL6UBd1Lbe9QjRjBx06Zhn975+EDfl+mvhjSiuCKcw9HKkHo7PFX40oLIKu4fYLo6LbPj5sdpLPwvDn44jmzkDYEY6XwPWN2OzZcdT6+fcFcPPzVBThn7nj3mL997lQ0F2kR3PSJE93XJx82BnddfioAe4YuZ/IXHT/JGUt+fFe+50j3tYyLqAro7CPGY/xwOwtKXkcK2uljW3Hfl87AqNYGtLc14v4vn4GjDh2B+798BmaOa3Pvf9flp+KkGWPca5LHIsi/9iqC0McNpdT2DtWIEVz5niNx+VkzB/7GTF0xpBUBEWFEc9qoCIQbLPa6hiqBdMGkDZJEtkTIGgLOUnBJAS0tguEFKAJ91k9EbsaQjB3IbKEgi0V+NOp9kzH8+YUStEiN6hqKG5g3X780OEbADFWGtCIA7KCrSRFIdNdQJWhIBWfdtDjulqyhyEzPxpGumZHN+QBxWLdNwLz8o60I8hZBwlU44YJOVTrJhFplXB4BGZQp1KcI//4yrM9c7OpgnD3KDFWqogiI6FwiWkNErxPRwkrea0RzGl1hriEntz6iM7SRuOv5ui0XDIJWVvNmDAPQWyLIOgJVIEf5zE2T9VSC0JhK+hZSN7muVNQMJXXGXi4BGeSDz2TzgrsURSA/q2J1PtcRMEOVAVcERJQE8CsA5wGYC+BSIppbqfuNaE67Pn6VfIzAsQiKmCXGbdIlFYFJiMlGb2EWgZzBmuoIojApipRjEcj0UfnoUS0kRmiuoXLbUEFyVnUH6ZXKxV2fLQKGUamGRXAigNeFEOuEEP0AbgVwYaVuNqI5jT0H+pDNWejpz6HX6cvvKgBLxG4lrTO2La5FYEuQHkMhWbMj3E2+byl3LDdrx28RFEMqmUBTOumme8r22lGrfbUq/X/K0ZtfJ8i6URVo0HoPsa5f9Jk2bBEwQ5Vq1BEcCkBNpN8M4KRK3WxUSwO2dvZi5jfvNe5/8vXd+MzNS4u69ujWeBaBzPYxdfcc4ywOP2V0i2/fjPZhWLf7oKsAZJO40TFdUkE0p5NobUyh3bFopEIztaEGbEuhP2eBiDB5dDM27bXXP5jZPgy79vdh9nj7r2mpyzhMHdOCjXuC14KQ2UmA3XOpWKaOacXSNztC1ywIgyuLmaFKzRaUEdFlAC4DgClTphR9nU+eNt1tYfyOuePx+Gu7PEs7rtrW5TvnPy88Erct2YxXtnR6tl9x7hyMaW3AXcu24Ok39iCdJHzqtOn47ZPr3f3jhzdi2thWbNnXg76s3Wtn7sThAIC3zx2PX/6/4zBnQhu+9OdlWLGlC9PHtuKGj83H/Gn+LqI/veRYPL9+LyaNspXEgsPH4X8/dDyOmNjmOe7hr56J3z+1Hn989k2cPGMMFhzejh/cuzrwM/mfS+dh/PAmtLc14oSpo/D2ueMxd+IIHHXocDz45TN83VEf+dqZ7hoJt3/mFLy+4wAA4NcfPgEvbdqH+dNG45k39uCw9mGB9wzjr58+Ba/tCG6l/bm3zcLM8W1IEoWuNxDF9993FN519ETMmTC8qPNLqWFgmFqmGopgC4DJyvtJzjYPQojrAVwPAPPnzy/aHT1ZmWmfd/QEPL9hr0cR7DnY7zvnIydPAxF5FEEqQfjMgsMA2MtFPv3GHvRlLCw4fJyrCD595gxXWOhrGgC2ILngGHs9g9NntWPFli6kkgm8bc5437EA0NaUxtlH5Pc1pBJ419ETfccd1j4Mx00ehT8++yYmjmgyKhWVE6bm95/nXO/8Y+y/s8a3+Y6fNKrFVUbj2powrs1eNGZESxoLDh8HwFZyxdLe1oj2kP78DakE3nPsIYH749LSkCppnAwzVKlGjOAFALOIaDoRNQD4IIC7B+LGJt/63gN+RQB42y8D3kChvM7B/qwn5bKQGaP0t5eSF68is3iylmAXBsMwBTHgFoEQIktEnwNwP4AkgBuEECsH4t4jmv2+9f2GxeAB+Fo0q+4SWWHb3Z+LTLkMQvrjuwPuXyhuPyEhKrs4C8MwQ46qxAiEEPcAuGeg71tItk1TiICX1+nuz7rVuYUiF4WJ05I6DrLS17IEZ7cwDFMQQ76yWKUQRdCYDs6AySuCHNKp4oRus7QIyrRGrLQCchZbBAzDFAYrggDCLIKWRunWycXqKGq8hqNoyuUaYkXAMEyx1IUiuOyMGQDs7JPLF/g7OU4e3ey+lm2b9YXc1XbOsrXEJ06bFlmNG8TRTjvo9zudP0vlyEPslMiLTpg0pFxD7zq6+HTRcvGRk6dWewgMU1Go2AZcA8n8+fPFkiVLynpNyxKY8Q07TLHqP8/1Cf6New7izB8/CgBY/b1zkU76l3YEgG2dPTj5B48AADZcdX5Zx1gs63cfxFlXPwqgdsbEMMzAQ0QvCiHmRx1XswVllUZtkaArAcDrRtIziFSKdQ1VEnexeXYRMQwTg9qTYjVCW1O8eEJtKgJnjeEiU1sZhqkvWFIEEDfgWmyMoJLIsbNFwDBMHGpPig0y9BXAagGpCNgiYBgmDiwpSqQWZ905t600f70Mw0TDkqJEarEjpVQEw5vrNheAYZgCqGtJ8YP3H43Z44NbJ9/48bdgf290wdd33j0XJ04P7/g5kEwa1YyvvH023nfcodUeCsMwg4C6rSNgGIYZ6sStI2DXEMMwTJ3DioBhGKbOYUXAMAxT57AiYBiGqXNYETAMw9Q5rAgYhmHqHFYEDMMwdQ4rAoZhmDpnUBSUEdEuABuLPH0sgN1lHE41GUrPAgyt5+FnqU2G0rMAhT/PVCFEe9RBg0IRlAIRLYlTWTcYGErPAgyt5+FnqU2G0rMAlXsedg0xDMPUOawIGIZh6px6UATXV3sAZWQoPQswtJ6Hn6U2GUrPAlToeYZ8jIBhGIYJpx4sAoZhGCaEIa0IiOhcIlpDRK8T0cJqjycKIrqBiHYS0Qpl22giepCIXnP+jnK2ExH9j/Nsy4no+OqN3A8RTSaixUT0KhGtJKIvOtsH3fMQURMRPU9ELzvP8l1n+3Qies4Z85+JqMHZ3ui8f93ZP62a4zdBREkieomI/uG8H8zPsoGIXiGiZUS0xNk26H5nAEBEI4nor0S0mohWEdHJA/EsQ1YREFESwK8AnAdgLoBLiWhudUcVyY0AztW2LQTwsBBiFoCHnfeA/VyznH+XAbh2gMYYlyyArwoh5gJ4K4DLnc9/MD5PH4C3CSGOBTAPwLlE9FYAPwTwUyHETAD7AHzSOf6TAPY523/qHFdrfBHAKuX9YH4WADhLCDFPSa0cjL8zAPg5gPuEEHMAHAv7O6r8swghhuQ/ACcDuF95/3UAX6/2uGKMexqAFcr7NQAmOq8nAljjvL4OwKWm42rxH4C/AXj7YH8eAC0AlgI4CXZhT0r/vQG4H8DJzuuUcxxVe+zKM0xyBMrbAPwDAA3WZ3HGtQHAWG3boPudARgBYL3++Q7EswxZiwDAoQA2Ke83O9sGG+OFENuc19sBjHdeD5rnc9wJxwF4DoP0eRxXyjIAOwE8COANAB1CCLmotTpe91mc/Z0AxgzsiEP5GYB/B2A578dg8D4LAAgADxDRi0R0mbNtMP7OpgPYBeD3jtvut0TUigF4lqGsCIYcwlb7gyrNi4iGAbgdwJeEEF3qvsH0PEKInBBiHuzZ9IkA5lR5SEVBRBcA2CmEeLHaYykjpwkhjoftKrmciM5Qdw6i31kKwPEArhVCHAfgIPJuIACVe5ahrAi2AJisvJ/kbBts7CCiiQDg/N3pbK/55yOiNGwlcLMQ4g5n86B9HgAQQnQAWAzbfTKSiFLOLnW87rM4+0cA2DPAQw3iVADvIaINAG6F7R76OQbnswAAhBBbnL87AdwJW1EPxt/ZZgCbhRDPOe//ClsxVPxZhrIieAHALCcbogHABwHcXeUxFcPdAD7qvP4obF+73P4RJ3PgrQA6FfOx6hARAfgdgFVCiGuUXYPueYionYhGOq+bYcc6VsFWCBc7h+nPIp/xYgCPODO5qiOE+LoQYpIQYhrs/xOPCCE+hEH4LABARK1E1CZfA3gHgBUYhL8zIcR2AJuI6HBn09kAXsVAPEu1AyQVDr68C8Ba2P7cb1Z7PDHGewuAbQAysGcHn4Ttj30YwGsAHgIw2jmWYGdFvQHgFQDzqz1+7VlOg23CLgewzPn3rsH4PACOAfCS8ywrAPyHs30GgOcBvA7gNgCNzvYm5/3rzv4Z1X6GgOdaAOAfg/lZnHG/7PxbKf+fD8bfmTO+eQCWOL+1uwCMGohn4cpihmGYOmcou4YYhmGYGLAiYBiGqXNYETAMw9Q5rAgYhmHqHFYEDMMwdQ4rAmZIQ0Q5pyul/BfahZaIPk1EHynDfTcQ0dgiznsnEX3X6Th5b6njYJg4pKIPYZhBTY+wW0PEQgjx60oOJganwy7uOh3Ak1UeC1MnsEXA1CXOjP1HTh/754loprP9SiL6mvP6C2Svp7CciG51to0morucbc8S0THO9jFE9ADZ6xX8Fnaxj7zXh517LCOi65wW6fp4LnGa2n0BdlO43wD4OBENxmp4ZpDBioAZ6jRrrqFLlH2dQoijAfwStvDVWQjgOCHEMQA+7Wz7LoCXnG3fAHCTs/07AJ4UQhwJu9/NFAAgoiMAXALgVMcyyQH4kH4jIcSfYXdoXeGM6RXn3u8p5eEZJg7sGmKGOmGuoVuUvz817F8O4GYiugt2uT9gt864CACEEI84lsBwAGcAeL+zfRER7XOOPxvACQBesNsvoRn5pmE6swGsc163CiH2x3g+hikZVgRMPSMCXkvOhy3g3w3gm0R0dBH3IAB/EEJ8PfQge4nFsQBSRPQqgImOq+jzQognirgvw8SGXUNMPXOJ8vcZdQcRJQBMFkIsBnAF7PbLwwA8Ace1Q0QLAOwW9joLjwP4f87282A3CwPsZmEXE9E4Z99oIpqqD0TYSywuAnAhgB/Bbp42j5UAMxCwRcAMdZqdmbXkPiGETCEdRUTLYa9JfKl2XhLAH4loBOxZ/f8IITqI6EoANzjndSPfHvi7AG4hopUAngbwJgAIIV4lom/BXkErAbuz7OUANhrGejzsYPFnAVxj2M8wFYG7jzJ1ibMwy3whxO5qj4Vhqg27hhiGYeoctggYhmHqHLYIGIZh6hxWBAzDMHUOKwKGYZg6hxUBwzBMncOKgGEYps5hRcAwDFPn/H9VzvPO26kvqQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def dqn(n_episodes=2000, max_t=1000, eps_start=1.0, eps_end=0.01, eps_decay=0.995, target_scores=13.0):\n",
    "    \"\"\"Deep Q-Learning.\n",
    "    \n",
    "    Params\n",
    "    ======\n",
    "        n_episodes (int): maximum number of training episodes\n",
    "        max_t (int): maximum number of timesteps per episode\n",
    "        eps_start (float): starting value of epsilon, for epsilon-greedy action selection\n",
    "        eps_end (float): minimum value of epsilon\n",
    "        eps_decay (float): multiplicative factor (per episode) for decreasing epsilon\n",
    "        target_scores (float): average scores aming to achieve, the agent will stop training once it reaches this scores\n",
    "    \"\"\"\n",
    "    start = time.time()                # Start time\n",
    "    scores = []                        # list containing scores from each episode\n",
    "    scores_window = deque(maxlen=100)  # last 100 scores\n",
    "    eps = eps_start                    # initialize epsilon\n",
    "    \n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        # Reset env and score at the beginning of episode\n",
    "        env_info = env.reset(train_mode=True)[brain_name] # reset the environment\n",
    "        state = env_info.vector_observations[0]            # get the current state\n",
    "        score = 0                                          # initialize the score\n",
    "        \n",
    "        for t in range(max_t):\n",
    "            action = agent.act(state, eps)\n",
    "            \n",
    "            env_info = env.step(action)[brain_name]        # send the action to the environment\n",
    "            next_state = env_info.vector_observations[0]   # get the next state\n",
    "            reward = env_info.rewards[0]                   # get the reward\n",
    "            done = env_info.local_done[0]                  # see if episode has finished\n",
    "            \n",
    "            agent.step(state, action, reward, next_state, done)\n",
    "            state = next_state\n",
    "            score += reward\n",
    "            if done:\n",
    "                break \n",
    "\n",
    "        scores_window.append(score)       # save most recent score\n",
    "        scores.append(score)              # save most recent score\n",
    "        eps = max(eps_end, eps_decay*eps) # decrease epsilon\n",
    "        \n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)), end=\"\")\n",
    "        \n",
    "        if i_episode % 100 == 0:\n",
    "            print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))\n",
    "        \n",
    "        if np.mean(scores_window)>=target_scores:\n",
    "            print('\\nEnvironment solved in {:d} episodes!\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))\n",
    "            torch.save(agent.qnetwork_local.state_dict(), model_weight_name)\n",
    "            break\n",
    "\n",
    "    time_elapsed = time.time() - start\n",
    "    print(\"Time Elapse: {:.2f}\".format(time_elapsed))\n",
    "    \n",
    "    return scores\n",
    "\n",
    "scores = dqn(n_episodes=1800, target_scores=13.0)\n",
    "\n",
    "# plot the scores\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(len(scores)), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Watch the agent running ( Using saved weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score: 16.0\n"
     ]
    }
   ],
   "source": [
    "# load the weights from file\n",
    "agent.qnetwork_local.load_state_dict(torch.load(model_weight_name))\n",
    "\n",
    "env_info = env.reset(train_mode=False)[brain_name] # reset the environment\n",
    "state = env_info.vector_observations[0]            # get the current state\n",
    "score = 0                                          # initialize the score\n",
    "\n",
    "while True:\n",
    "    action = agent.act(state)                      # select an action\n",
    "    env_info = env.step(action)[brain_name]        # send the action to the environment\n",
    "    next_state = env_info.vector_observations[0]   # get the next state\n",
    "    reward = env_info.rewards[0]                   # get the reward\n",
    "    done = env_info.local_done[0]                  # see if episode has finished\n",
    "    score += reward                                # update the score\n",
    "    state = next_state                             # roll over the state to next time step\n",
    "    if done:                                       # exit loop if episode finished\n",
    "        break\n",
    "    \n",
    "print(\"Score: {}\".format(score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Close Unity ML Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:deeprl]",
   "language": "python",
   "name": "conda-env-deeprl-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}