10_Renforcement_Learning_Moutain_Car/5_DQN_keras_mountain_car.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XT2oM6FHVJCf"
   },
   "source": [
    "## Enviroment: \n",
    "Open AI gym [MountainCar-v0](https://github.com/openai/gym/wiki/MountainCar-v0)\n",
    "\n",
    "### Observation\n",
    "Type: Box(2)\n",
    "\n",
    "| Num  | Observation | Min   | Max  |\n",
    "| ---- | ----------- | ----- | ---- |\n",
    "| 0    | position    | -1.2  | 0.6  |\n",
    "| 1    | velocity    | -0.07 | 0.07 |\n",
    "\n",
    "### Actions\n",
    "\n",
    "Type: Discrete(3)\n",
    "\n",
    "| Num  | Action     |\n",
    "| ---- | ---------- |\n",
    "| 0    | push left  |\n",
    "| 1    | no push    |\n",
    "| 2    | push right |\n",
    "\n",
    "### Reward\n",
    "\n",
    "-1 for each time step, until the goal position of 0.5 is reached. As with MountainCarContinuous v0, there is no penalty for climbing the left hill, which upon reached acts as a wall.\n",
    "\n",
    "### Starting State\n",
    "\n",
    "Random position from -0.6 to -0.4 with no velocity.\n",
    "\n",
    "### Episode Termination\n",
    "\n",
    "The episode ends when you reach 0.5 position, or if 200 iterations are reached.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xzVWtGxJVJCh"
   },
   "source": [
    "## 1. gym enviroment setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "G312rzBjVJCi",
    "outputId": "f24448be-40e1-4cdd-e008-15b2534e6d37"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.55733036,  0.        ])"
      ]
     },
     "execution_count": 1,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "env = gym.make(\"MountainCar-v0\")\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "80YCNcHUZYtI"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import models\n",
    "from tensorflow.keras import losses,optimizers,metrics\n",
    "from tensorflow.keras.layers import Dense, Activation, Flatten\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from collections import deque\n",
    "import random\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IjkusASudj9W"
   },
   "outputs": [],
   "source": [
    "ACTION_SPACE_SIZE = env.action_space.n\n",
    "REPLAY_MEMORY_SIZE = 50_000\n",
    "MIN_REPLAY_MEMORY_SIZE = 1_000\n",
    "MINIBATCH_SIZE = 64\n",
    "UPDATE_TARGET_EVERY = 5\n",
    "DISCOUNT = 0.99\n",
    "EPISODES =1000\n",
    "\n",
    "# Exploration settings\n",
    "epsilon = 1  # not a constant, going to be decayed\n",
    "EPSILON_DECAY = 0.995\n",
    "MIN_EPSILON = 0.001\n",
    "\n",
    "ep_rewards = []\n",
    "\n",
    "AGGREGATE_STATS_EVERY = 50\n",
    "MIN_EPSILON = 0.001"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Gm5w81XdRgyz"
   },
   "source": [
    "### Plot epsilon decay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 286
    },
    "colab_type": "code",
    "id": "41yF0b4u46gb",
    "outputId": "03888fbc-7567-49b7-f190-4910f8c40816"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f683078e4e0>]"
      ]
     },
     "execution_count": 24,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xt8VOWdx/HPb2YyuZMQEgIkgXAJ\nSkRAGi+ILF2v6Cq23l27arV1bWutbV/tanfrdt19rdt1V2u71mqtbW2t1norRVu81NZLUQwKyEUg\n3AOYhFsSQq4zz/4xAwaKJoRJTubM9/16zYs5z3mS+Z0c/ebkOec8x5xziIiIvwS8LkBERBJP4S4i\n4kMKdxERH1K4i4j4kMJdRMSHFO4iIj6kcBcR8SGFu4iIDyncRUR8KOTVBxcWFrry8nKvPl5EJCkt\nXrx4h3OuqKd+noV7eXk51dXVXn28iEhSMrNNvemnYRkRER9SuIuI+JDCXUTEhxTuIiI+1GO4m9nD\nZlZvZss/Yr2Z2ffNrMbMlpnZ9MSXKSIiR6I3R+4/A+Z8zPpzgYr46wbg/qMvS0REjkaP4e6cexXY\n9TFdLgQecTFvAvlmNjJRBYqIyJFLxJh7CbCl23JtvO2vmNkNZlZtZtUNDQ19+rB3Nu/mu394Hz0e\nUETkow3oCVXn3IPOuSrnXFVRUY83WB3W8q2N3P+ndaxr2Jvg6kRE/CMR4b4VKOu2XBpv6xdnVRYD\nsGBFXX99hIhI0ktEuM8Dro5fNXMK0Oic256A73tYI/MymVqaxwsrPuivjxARSXq9uRTyMWAhcIyZ\n1ZrZ9WZ2o5ndGO/yPLAeqAF+DHyx36qNO/u4ESytbWR7Y2t/f5SISFLqceIw59yVPax3wJcSVlEv\nnHPcCO5asJoXV9Zx9YzygfxoEZGkkJR3qE4YnsO4omwWaGhGROSwkjLcIXb0/ub6XezZ1+F1KSIi\ng07ShvvZlcVEoo4/vl/vdSkiIoNO0ob71NJ8ioeka2hGROQwkjbcAwHjrMpi/rymgdaOiNfliIgM\nKkkb7hAbd2/rjPLa2r5NZSAi4ldJHe4njx1GbkZId6uKiBwiqcM9HApw1qRiXlz5AR1dUa/LEREZ\nNJI63AH+bspImtq6eKNmh9eliIgMGkkf7qdVFJKbEWL+sn6bzkZEJOkkfbinh4KcXTmCFzQ0IyJy\nQNKHO8DfTRlBc1sXr9foqhkREfBJuJ82oUhDMyIi3fgi3MOhAOccN4IXV9TR3qUbmkREfBHuELtq\nprm9i9fX6qoZERHfhPvM8YXkZabxnIZmRET8E+7hUICzK4t5caWGZkREfBPu8OHQzGtrNDQjIqnN\nV+E+c0Ih+Vlp/G7ZNq9LERHxlK/CPS0Y4NzJI3lhRR0t7V1elyMi4hlfhTvAp6aNorUzwosrNVOk\niKQu34X7ieUFlORn8uySrV6XIiLiGd+FeyBgXDhtFK+t3cGOve1elyMi4gnfhTvAp04oIRJ1zF+q\nE6sikpp8Ge4Ti3OpHDmEZ5Yo3EUkNfky3AE+dcIolm7Zw4YdLV6XIiIy4Hwb7nOnlmAGz76rE6si\nknp8G+4j8jI4dfwwnl2yFeec1+WIiAwo34Y7wIXTSti0cx9LtuzxuhQRkQHl63CfM3kE6aEAz2ho\nRkRSjK/DfUhGGmcfN4LfLtlGW6dmihSR1NGrcDezOWa22sxqzOzWw6wfbWavmNm7ZrbMzM5LfKl9\nc1lVKY2tnby0StMRiEjq6DHczSwI3AecC1QCV5pZ5SHd/gV4wjl3AnAF8MNEF9pXp44vpCQ/kyeq\na70uRURkwPTmyP0koMY5t9451wE8Dlx4SB8HDIm/zwMGzd1DwYBx8fQSXlvbwLY9rV6XIyIyIHoT\n7iXAlm7LtfG27r4DfMbMaoHngS8f7huZ2Q1mVm1m1Q0NDX0ot28u+UQZzsHT7+joXURSQ6JOqF4J\n/Mw5VwqcB/zCzP7qezvnHnTOVTnnqoqKihL00T0bPSyLGeOG8UR1LdGornkXEf/rTbhvBcq6LZfG\n27q7HngCwDm3EMgAChNRYKJcWlXK5l37WLRxl9eliIj0u96E+9tAhZmNNbMwsROm8w7psxk4A8DM\nJhEL94Ebd+mFcyePJCc9xG90YlVEUkCP4e6c6wJuAhYAq4hdFbPCzO4ws7nxbl8HPm9mS4HHgGvd\nILvnPzMc5IKpo3j+ve00t3V6XY6ISL8K9aaTc+55YidKu7fd3u39SmBmYktLvEurSnls0WaeW7ad\nK04a7XU5IiL9xtd3qB7qhLJ8Kobn8NiizV6XIiLSr1Iq3M2Mq04ezdLaRt6rbfS6HBGRfpNS4Q7w\n6emlZKYF+dWiTV6XIiLSb1Iu3PMy05g7dRTPvruNJp1YFRGfSrlwB7jqlNG0dkb0lCYR8a2UDPcp\npfkcX5LHo29u1lOaRMSXUjLcAT5zymhW1zVTvWm316WIiCRcyob7BVNHkZse4tE3dWJVRPwnZcM9\nKxziouklPP/eB+xq6fC6HBGRhErZcAe46pQxdESiPFG9pefOIiJJJKXDfWJxLiePLeAXCzfRFYl6\nXY6ISMKkdLgDXHfaWLbuaeXFlXrGqoj4R8qH+5mTiikryOSnb2z0uhQRkYRJ+XAPBoxrZpSzaOMu\nlm/VfDMi4g8pH+4Al51YRnY4yMNvbPC6FBGRhFC4A0My0rjkE6XMX7qdhuZ2r8sRETlqCve4a04t\npyMS5dG3dFOTiCQ/hXvcuKIcTj92OL98cxPtXRGvyxEROSoK924+O7OcHXs7+N3S7V6XIiJyVBTu\n3Zw2oZBjinP58avrNVukiCQ1hXs3ZsY/zh7H6rpmXlld73U5IiJ9pnA/xAVTRzEqL4Mf/Xm916WI\niPSZwv0QacEA188ax6INu3hns+Z6F5HkpHA/jCtOLCMvM40f/Wmd16WIiPSJwv0wstNDXDNjDC+u\nqqOmfq/X5YiIHDGF+0e4+tRywsEAD76qo3cRST4K949QmJPOZVVlPPPuVj5obPO6HBGRI6Jw/xif\nnzWOSNTx49d05YyIJBeF+8cYPSyLT00r4dG3NmlCMRFJKgr3Htx0+gQ6uqI8pKN3EUkiCvcejCvK\nYe7UUTyycBM79+roXUSSQ6/C3czmmNlqM6sxs1s/os9lZrbSzFaY2a8SW6a3bjq9grauCD9+TQ/z\nEJHk0GO4m1kQuA84F6gErjSzykP6VAC3ATOdc8cBt/RDrZ6ZMDyHC6aM4pGFG9nV0uF1OSIiPerN\nkftJQI1zbr1zrgN4HLjwkD6fB+5zzu0GcM75btatL58+gdbOCD95XWPvIjL49SbcS4At3ZZr423d\nTQQmmtkbZvammc053DcysxvMrNrMqhsaGvpWsUcqinM57/iR/Pwvm9izT0fvIjK4JeqEagioAD4J\nXAn82MzyD+3knHvQOVflnKsqKipK0EcPnJtPr6Clo4sHX9XRu4gMbr0J961AWbfl0nhbd7XAPOdc\np3NuA7CGWNj7yjEjcpk7dRQ/fWMj9U26a1VEBq/ehPvbQIWZjTWzMHAFMO+QPs8SO2rHzAqJDdP4\n8vD2a2dNpDMS5Qd/rPG6FBGRj9RjuDvnuoCbgAXAKuAJ59wKM7vDzObGuy0AdprZSuAV4BvOuZ39\nVbSXxgzL5vITy3hs0WY279zndTkiIodlXj0rtKqqylVXV3vy2UerrqmN2Xe9wrmTR3LP5dO8LkdE\nUoiZLXbOVfXUT3eo9kHxkAyuPXUszy7ZyvsfNHldjojIX1G499EXZo8nJz3E/yxY7XUpIiJ/ReHe\nR3lZadw4ezwvraqneuMur8sRETmIwv0ofHZmOcNz0/mP51YRjXpz7kJE5HAU7kchKxziG+ccw5It\ne/jdsm1elyMicoDC/ShdPL2U40YN4b//sJq2zojX5YiIAAr3oxYIGP/yd5Vs3dPKT17XlMAiMjgo\n3BNgxvhhnF1ZzA9fqaG+WdMSiIj3FO4J8q3zJtERiXL3C2u8LkVEROGeKOWF2Vwzo5xfV29h5Tbd\n2CQi3lK4J9CXT69gaFaY23+7HK+mdRARAYV7QuVlpXHrucdSvWk3T79z6KzIIiIDR+GeYJdML2X6\n6Hzu/P0qGls7vS5HRFKUwj3BAgHjjgsns6ulg3te1MlVEfGGwr0fTC7J4x9OGcMjCzeyYluj1+WI\nSApSuPeTr519DEOzwnz72eWad0ZEBpzCvZ/kZaZx23mTeGfzHn5dvcXrckQkxSjc+9HF00s4ZVwB\n//n8Kur0QG0RGUAK935kZtx50RQ6uqLc/tvlXpcjIilE4d7PxhZmc8uZE1mwoo4/LN/udTkikiIU\n7gPgc7PGUjlyCN/+7Qoa9+nadxHpfwr3AZAWDPDfl0xhV0sHd/5+ldfliEgKULgPkMkleXxu1lge\nf3sLr6/d4XU5IuJzCvcB9NUzJzKuKJtvPLlUUxOISL9SuA+gjLQgd182jfrmdv7tdyu8LkdEfEzh\nPsCmleXzpb+dwNPvbNXVMyLSbxTuHvjy6ROYXDKEbz2znIbmdq/LEREfUrh7IC0Y4J7LprG3vYtv\nPfOeHuwhIgmncPdIRXEu3zznGF5cWcdjizT3jIgklsLdQ9fNHMusikL+7XcrWFPX7HU5IuIjCncP\nBQLG3ZdNIzcjjZt+9Q6tHRGvSxIRn+hVuJvZHDNbbWY1Znbrx/S72MycmVUlrkR/K8pN5+7LprKm\nbi93zF/pdTki4hM9hruZBYH7gHOBSuBKM6s8TL9c4CvAW4ku0u/+ZmIRN84ez2OLNjN/2TavyxER\nH+jNkftJQI1zbr1zrgN4HLjwMP3+HfguoInL++DrZ09kWlk+tz31Hpt37vO6HBFJcr0J9xKg++Uc\ntfG2A8xsOlDmnHvu476Rmd1gZtVmVt3Q0HDExfpZWjDAD648AQy++KvFtHVq/F1E+u6oT6iaWQC4\nG/h6T32dcw8656qcc1VFRUVH+9G+U1aQxfcun8byrU388zPLdf27iPRZb8J9K1DWbbk03rZfLjAZ\n+JOZbQROAebppGrfnDGpmJvPqOCpd2r55VubvS5HRJJUb8L9baDCzMaaWRi4Api3f6VzrtE5V+ic\nK3fOlQNvAnOdc9X9UnEKuOWMCv72mCLu+N0KFm/a7XU5IpKEegx351wXcBOwAFgFPOGcW2Fmd5jZ\n3P4uMBUFAsb3Lj+BkXmZfPHRxdQ36xy1iBwZ82pct6qqylVX6+D+46zc1sRF97/B8SV5/PJzJ5Me\nCnpdkoh4zMwWO+d6HPbWHaqDWOWoIdx1yVTe3rib257WBGMi0nshrwuQj3fB1FGsb2jhnpfWML4o\nhy/97QSvSxKRJKBwTwI3nzGB9Tv2cteC1YwtzOa840d6XZKIDHIalkkCZsZ3L57C9NH5fO2JJSzd\nssfrkkRkkFO4J4mMtCAPXl1FYU46n3ukmi27NEWBiHw0hXsSKcxJ56fXnkh7Z4SrH17Ezr16RJ+I\nHJ7CPclUFOfy8LUnsm1PK9f97G1a2ru8LklEBiGFexKqKi/gvr+fzvJtTdz4y8V0dEW9LklEBhmF\ne5I6s7KYOz99PK+t3cE3nlxKNKpr4EXkQ7oUMolddmIZDXvbuWvBanIzQvz7hZMxM6/LEpFBQOGe\n5L74yfE0tXXywJ/XEw4G+fb5kxTwIqJwT3Zmxq1zjqWjK8rDb2wgHArwT3OOUcCLpDiFuw+YGbef\nX0lnJMqP/ryO9FCAr5410euyRMRDCnefMDPumDuZjq4o9768lmDA+PLpE3QEL5KiFO4+EggYd140\nhUgU7n5xDa2dEb55joZoRFKRwt1nggHjrkumkJEW4P4/raO1I8Lt51cSCCjgRVKJwt2HAgHjPz41\nmaxwkB+/toHWjgj/edHxBBXwIilD4e5TZsa3zptEZjjE919ey77OCP976VTCId23JpIKFO4+ZmZ8\n7ayJZIWD/Nfv32fPvg5+eNV0cjPSvC5NRPqZDuNSwI2zx3PXJVP4y7qdXP7Am9Q16YHbIn6ncE8R\nl1aV8ZNrqti4s4WLfvgXauqbvS5JRPqRwj2FfPKY4fz6hhm0d0W5+P6FLNqwy+uSRKSfKNxTzPGl\neTzzxVMZlh3mqofe5Im3t3hdkoj0A4V7CioryOKZL87k5LHD+OZTy/j3+SvpimhOeBE/UbinqLys\nNH722RO59tRyfvL6Bq77eTWNrZ1elyUiCaJwT2GhYIDvzD2OOy86noXrdvDp+95gbZ1OtIr4gcJd\nuPKk0Tz6uVNoautk7v+9wbPvbvW6JBE5Sgp3AeCksQU8d/Msji/J45ZfL+G2p9+jrTPidVki0kcK\ndzmgeEgGv/r8ydw4ezyPLdrMxff/hU07W7wuS0T6QOEuBwkFA9x67rE8dHUVW3bt4/wfvM5vl2iY\nRiTZKNzlsM6sLOa5m2dRMTyHrzy+hK88/q6uphFJIr0KdzObY2arzazGzG49zPqvmdlKM1tmZi+b\n2ZjElyoDrawgiyf+cQZfO2si85dt59zvvcrCdTu9LktEeqHHcDezIHAfcC5QCVxpZpWHdHsXqHLO\nTQGeBP470YWKN0LBADefUcFTXziV9LQgf//Qm9z5/CqdbBUZ5Hpz5H4SUOOcW++c6wAeBy7s3sE5\n94pzbl988U2gNLFlitemleXz3M2nceVJo3ng1fWcd+9rmptGZBDrTbiXAN0nIKmNt32U64HfH26F\nmd1gZtVmVt3Q0ND7KmVQyAqH+M9PH88vrz+ZjkiUyx5YyLefXU5zm8biRQabhJ5QNbPPAFXAXYdb\n75x70DlX5ZyrKioqSuRHywA6raKQF776N1w3cyy/fGsT59zzKn98v87rskSkm96E+1agrNtyabzt\nIGZ2JvDPwFznXHtiypPBKisc4vYLKnnqC6eSnR7iup9V8/lHqtmya1/PXywi/a434f42UGFmY80s\nDFwBzOvewcxOAB4gFuz1iS9TBqvpo4fy3M2z+Kc5x/JGzQ7OvPvP3PvSWp1wFfFYj+HunOsCbgIW\nAKuAJ5xzK8zsDjObG+92F5AD/MbMlpjZvI/4duJD4VCAL3xyPC9/fTZnVhZzz0trOPueV3lpZR3O\nOa/LE0lJ5tX/fFVVVa66utqTz5b+9UbNDv513gpq6vdy6vhhfOu8SUwuyfO6LBFfMLPFzrmqnvrp\nDlVJuJkTCnn+5ll854JKVm1v4vwfvM4tj7+r8XiRAaQjd+lXTW2d/OhP6/jJ6xtwDq6dWc6Ns8dT\nkB32ujSRpNTbI3eFuwyI7Y2t/O8La3jqnVqy0oJcc2o5n581jqEKeZEjonCXQWltXTP3vryW597b\nTlZakGtnxkI+P0shL9IbCncZ1NbEQ/7597aTHQ5x9YwxXDuznOG5GV6XJjKoKdwlKaz+oJnv/zEW\n8mmBABdNL+Fzs8YxYXiO16WJDEoKd0kqG3e08NDr6/lNdS3tXVHOnFTMjbPHUVVe4HVpIoOKwl2S\n0o697TyycBO/WLiR3fs6mVKaxz+cMoYLpo4iIy3odXkinlO4S1Lb19HFU4tr+fnCTdTU7yU/K43L\nqsq46uTRjBmW7XV5Ip5RuIsvOOd4c/0ufvHmRhasqCPqHLMnFnF5VRlnTComHNJ9eJJaFO7iOx80\ntvHYos08/vZm6praGZqVxoXTSri0qpTjRml6A0kNCnfxra5IlNdqdvDk4lpeXFFHRyTKpJFDuPQT\npVwwdRRFuelelyjSbxTukhL27Otg3tJtPLm4lmW1jQQMThk3jAumjmLOcSN0B6z4jsJdUs6aumbm\nL93G/GXbWb+jhVDAmDmhkPOnjOTs40aQl5nmdYkiR03hLinLOceKbU3MX7ad+cu2Ubu7lVDAOHlc\nAWdOKubMScWUFWR5XaZInyjcRYgF/dLaRhas+ICXVtaxtn4vAMeOyOWsyljQH1+SRyBgHlcq0jsK\nd5HD2LCjhZdX1fHCyjqqN+4i6mBYdpjTKgqZVVHErIpCiodofhsZvBTuIj3Y3dLBK6vreXVNA6/X\n7GDH3g4AjinOZVZFIbMmFnFi+VCywiGPKxX5kMJd5AhEo45VHzTx2todvL52B4s27qKjK0ooYBxf\nmsdJYws4eWwBnxhToBOz4imFu8hRaO2IsGjjLt5av5NFG3axtHYPnRGHGUwaMYSTxhZw0tgCThid\nz4ghGZhpzF4GhsJdJIHaOiO8u3kPizbsYtHGnSzetJu2zigAw3PTmVaWz7TR+Uwrzef40jxyM3R0\nL/2jt+GuwUSRXshICzJj/DBmjB8GVNDRFWXFtkaWbtnDki17WFrbyAsr6wAwg4rhOUwpzee4UUOY\nNDL20nCODCSFu0gfhEMBThg9lBNGDz3QtmdfB0trG1myeQ9La/fwp9X1PLm49sD60qGZVMaDvnLU\nECpHDqF0aKaGdKRfKNxFEiQ/K8zsiUXMnlh0oK2+uY2V25pYub2JVdubWbmtkRdX1bF/NDQ7HGT8\n8BwmFOUwfngOFcNzmDA8h9EFWYSCmvFS+k7hLtKPhudmMPyYDD55zPADba0dEVbXNbNyWxNr6ppZ\n17CXhet38vS7Ww/0CQcDlBdmMWF4DuMKcxg9LIvyYdmMGZbF8Nx0He1LjxTuIgMsMxyMnYAtyz+o\nvbmtk3UNLdTU72VtfTPr6veyclsTC1bUEYl+eOFDZlqQ0QVZjB6WxZiCLMYUZjOmIIvSoZmMys/U\nE6sEULiLDBq5GWmHDf3OSJStu1vZtGsfm3a2sGnnPjbt3MfGHS28uqaB9q7oQf2HZYcZlZ/JyLwM\nRuVnUpIfC/1R+RmU5GdSmJOu6RZSgMJdZJBLCwYoL8ymvDAbKDpoXTTqqG9uZ9POFrbuaWXbnla2\nNbaxbU8rG3e28EbNDlo6Igd9TShgFOWmMzw3naLcjAPvhw9Jjw0j5aZTFH+ladw/aSncRZJYIGCM\nyMtgRN7h58NxztHU1hUL/W7hX9/UTn1zG7W79/Hu5t3sbOk47NcXZIcpzAkzNCvMsP3/ZocZmh2m\nIP7qvk5DQoOHwl3Ex8yMvMw08jLTmDRyyEf264xE2bG3PR767TQ0x8K/vrmdnXvb2d3SyeoPmtm9\nr5Pd+zr4qHsfs8NBhmaHyc9KY0hG/JUZIi9z//tYLUMyQwcvZ6SRkRbQieIEUriLCGnBACPzMhmZ\nl9lj30jU0djaya6WjgOv3fs6DlpubO2kqbWT9Tv20tTaRWNrJ62dkY/9vmlBIzcjjez0INnhENnp\nsVdOepCscIic9BDZB70PkR0OdusXIiscJCMtSGY4SEYokNKXk/Yq3M1sDnAvEAQecs791yHr04FH\ngE8AO4HLnXMbE1uqiAwGwYAdGJI5Eh1dUZrbOmlq6zoQ/k1tnQfCv6mtk+a2Tva1R9jb3kVLR6x9\n+55WWtq74m2Rg64c6kla0MgIBckIB8lIC5CZFgv//a/MtED83+5tsb7poQBpoQDhYIBwKLYcDgUI\nB4OEQwHSgvZhe7yte3s46O1fIj2Gu5kFgfuAs4Ba4G0zm+ecW9mt2/XAbufcBDO7AvgucHl/FCwi\nySkcCjAsJ51hOX1/gLlzjvauKC3tXezriP8SiId+S/x9W1eUto4IbZ0RWuOvts4obZ0ftrV1Rmhs\n7aSuMUJbV4TWeP+2zigdkWjPhfR2m+O/GA4N/VvOnMgFU0cl7HMOpzdH7icBNc659QBm9jhwIdA9\n3C8EvhN//yTwf2ZmzqtZyUTEl8zswBH2sH76jEjU0dYZob0rSsf+V+SQf7uidEQidHS5g9u6InRE\nonRG3CFfHznwvjPqyM/q/3mGehPuJcCWbsu1wMkf1cc512VmjcAwYEf3TmZ2A3ADwOjRo/tYsohI\n/wkGLD6O73UlR2dAzzY45x50zlU556qKiop6/gIREemT3oT7VqCs23JpvO2wfcwsBOQRO7EqIiIe\n6E24vw1UmNlYMwsDVwDzDukzD7gm/v4S4I8abxcR8U6PY+7xMfSbgAXELoV82Dm3wszuAKqdc/OA\nnwC/MLMaYBexXwAiIuKRXl3n7px7Hnj+kLbbu71vAy5NbGkiItJXqXv7loiIjyncRUR8SOEuIuJD\n5tVFLWbWAGzq45cXcsgNUilA25watM2p4Wi2eYxzrscbhTwL96NhZtXOuSqv6xhI2ubUoG1ODQOx\nzRqWERHxIYW7iIgPJWu4P+h1AR7QNqcGbXNq6PdtTsoxdxER+XjJeuQuIiIfI+nC3czmmNlqM6sx\ns1u9ridRzKzMzF4xs5VmtsLMvhJvLzCzF81sbfzfofF2M7Pvx38Oy8xsurdb0DdmFjSzd81sfnx5\nrJm9Fd+uX8cnq8PM0uPLNfH15V7W3Vdmlm9mT5rZ+2a2ysxmpMA+/mr8v+nlZvaYmWX4cT+b2cNm\nVm9my7u1HfG+NbNr4v3Xmtk1h/us3kiqcO/2yL9zgUrgSjOr9LaqhOkCvu6cqwROAb4U37ZbgZed\ncxXAy/FliP0MKuKvG4D7B77khPgKsKrb8neBe5xzE4DdxB7hCN0e5QjcE++XjO4F/uCcOxaYSmzb\nfbuPzawEuBmocs5NJjb54P5HcfptP/8MmHNI2xHtWzMrAP6V2AORTgL+df8vhCPmnEuaFzADWNBt\n+TbgNq/r6qdt/S2x59auBkbG20YCq+PvHwCu7Nb/QL9keRF7NsDLwOnAfMCI3dgROnR/E5uVdEb8\nfSjez7zehiPc3jxgw6F1+3wf739KW0F8v80HzvHrfgbKgeV93bfAlcAD3doP6nckr6Q6cufwj/wr\n8aiWfhP/U/QE4C2g2Dm3Pb7qA6A4/t4PP4vvAd8E9j+ReBiwxznXFV/uvk0HPcoR2P8ox2QyFmgA\nfhofinrIzLLx8T52zm0F/gfYDGwntt8W4+/93N2R7tuE7fNkC3ffM7Mc4CngFudcU/d1Lvar3BeX\nN5nZ+UC9c26x17UMoBAwHbjfOXcC0MKHf6YD/trHAPEhhQuJ/WIbBWTz10MXKWGg922yhXtvHvmX\ntMwsjViwP+qcezreXGdmI+PrRwL18fZk/1nMBOaa2UbgcWJDM/cC+fFHNcLB2+SHRznWArXOubfi\ny08SC3u/7mOAM4ENzrkG51wn8DSxfe/n/dzdke7bhO3zZAv33jzyLymZmRF7otUq59zd3VZ1f4Th\nNcTG4ve3Xx0/634K0Njtz7/axVoVAAABEElEQVRBzzl3m3Ou1DlXTmw//tE5dxXwCrFHNcJfb29S\nP8rROfcBsMXMjok3nQGsxKf7OG4zcIqZZcX/G9+/zb7dz4c40n27ADjbzIbG/+o5O9525Lw+AdGH\nExbnAWuAdcA/e11PArfrNGJ/si0DlsRf5xEbb3wZWAu8BBTE+xuxK4fWAe8RuxrB8+3o47Z/Epgf\nfz8OWATUAL8B0uPtGfHlmvj6cV7X3cdtnQZUx/fzs8BQv+9j4N+A94HlwC+AdD/uZ+AxYucVOon9\nlXZ9X/YtcF18+2uAz/a1Ht2hKiLiQ8k2LCMiIr2gcBcR8SGFu4iIDyncRUR8SOEuIuJDCncRER9S\nuIuI+JDCXUTEh/4f95m5BOAGZZIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "recorder = {\"epsode\":[],\"epsilon\":[]}\n",
    "\n",
    "for epsode in range(EPISODES):\n",
    "    if epsilon > MIN_EPSILON:\n",
    "        epsilon *= EPSILON_DECAY\n",
    "        epsilon = max(MIN_EPSILON, epsilon)\n",
    "    \n",
    "    recorder[\"epsode\"].append(epsode)\n",
    "    recorder[\"epsilon\"].append(epsilon)\n",
    "\n",
    "plt.plot(recorder[\"epsode\"], recorder[\"epsilon\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fkddyPjSZOBU"
   },
   "source": [
    "## 2. Create Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tlX30pzsZXOb"
   },
   "outputs": [],
   "source": [
    "  def create_model():\n",
    "    model = models.Sequential()\n",
    "\n",
    "    model.add(Dense(16 ,input_shape=(env.observation_space.shape)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dense(16))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dense(16))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dense(ACTION_SPACE_SIZE))\n",
    "    model.add(Activation('linear'))\n",
    "    \n",
    "    print(model.summary())\n",
    "    \n",
    "    model.compile(loss = 'mse', optimizer=Adam(lr=0.001),metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lTRr9PZPZ1-1"
   },
   "source": [
    "## 2. Agent Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "w_7sLLVzZ1Tj"
   },
   "outputs": [],
   "source": [
    "class DQNAgent:\n",
    "\n",
    "  def __init__(self):\n",
    "    \n",
    "    # Replay memory\n",
    "    self.replay_memory = deque(maxlen=REPLAY_MEMORY_SIZE)\n",
    "    \n",
    "    # Prediction Network (the main Model)\n",
    "    self.model_prediction = create_model()\n",
    "    \n",
    "    # Target Network\n",
    "    self.model_target = create_model()\n",
    "    self.model_target.set_weights(self.model_prediction.get_weights())\n",
    "    \n",
    "   \n",
    "    \n",
    "    # Used to count when to update target network with prediction network's weights\n",
    "    self.target_update_counter = 0\n",
    "    \n",
    "    \n",
    "  \n",
    "  # Adds step's data to a memory replay array\n",
    "  # (current_state, action, reward, next_state, done)\n",
    "  def update_replay_memory(self, transition):\n",
    "    self.replay_memory.append(transition)\n",
    "  \n",
    "  \n",
    "  \n",
    "  # Queries prediction network for Q values given current observation space (environment state)\n",
    "  def get_qs(self, state):\n",
    "    return self.model_prediction.predict(np.array(state).reshape(-1, *state.shape))[0]\n",
    "  \n",
    "  def train(self, terminal_state, step):\n",
    "    \n",
    "    if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE:\n",
    "      return\n",
    "    \n",
    "    minibatch = random.sample(self.replay_memory, MINIBATCH_SIZE)\n",
    "    \n",
    "    # Get current states from minibatch, then query NN model_prediction for current Q values\n",
    "    current_states = np.array([transition[0] for transition in minibatch])\n",
    "    current_qs_list = self.model_prediction.predict(current_states)   \n",
    "    \n",
    "    # Get next_states from minibatch, then query NN model_target for target Q values\n",
    "    # When using target network, query it, otherwise main network should be queried\n",
    "    next_states = np.array([transition[3] for transition in minibatch])\n",
    "    target_qs_list = self.model_target.predict(next_states)\n",
    "    \n",
    "    X = []\n",
    "    y = []\n",
    "    \n",
    "    # Now we need to enumerate our batches\n",
    "    for index, (current_state, action, reward, next_state, done) in enumerate(minibatch):\n",
    "\n",
    "        # If not a terminal state, get new q from future states, otherwise set it to 0\n",
    "        # almost like with Q Learning, but we use just part of equation here\n",
    "        if not done:\n",
    "            max_target_q = np.max(target_qs_list[index])\n",
    "            new_q = reward + DISCOUNT * max_target_q\n",
    "        else:\n",
    "            new_q = reward\n",
    "\n",
    "        # Update Q value for given state\n",
    "        current_qs = current_qs_list[index]\n",
    "        current_qs[action] = new_q\n",
    "\n",
    "        # And append to our training data\n",
    "        X.append(current_state)\n",
    "        y.append(current_qs)    \n",
    "    \n",
    "    # Fit on all samples as one batch, log only on terminal state\n",
    "    self.model_prediction.fit(np.array(X), np.array(y), batch_size=MINIBATCH_SIZE, verbose=0, shuffle=False if terminal_state else None)\n",
    "    \n",
    "    # Update target network counter every episode\n",
    "    if terminal_state:\n",
    "        self.target_update_counter +=1\n",
    "    \n",
    "    # If counter reaches set value, update target network with weights of main network\n",
    "    if self.target_update_counter > UPDATE_TARGET_EVERY:\n",
    "      self.model_target.set_weights(self.model_prediction.get_weights())\n",
    "      self.target_update_counter = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 874
    },
    "colab_type": "code",
    "id": "TUDDhaxOadLa",
    "outputId": "4b30291d-f387-435a-e011-fa542d80db43"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_8 (Dense)              (None, 16)                48        \n",
      "_________________________________________________________________\n",
      "activation_8 (Activation)    (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 16)                272       \n",
      "_________________________________________________________________\n",
      "activation_9 (Activation)    (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 16)                272       \n",
      "_________________________________________________________________\n",
      "activation_10 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 3)                 51        \n",
      "_________________________________________________________________\n",
      "activation_11 (Activation)   (None, 3)                 0         \n",
      "=================================================================\n",
      "Total params: 643\n",
      "Trainable params: 643\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_12 (Dense)             (None, 16)                48        \n",
      "_________________________________________________________________\n",
      "activation_12 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 16)                272       \n",
      "_________________________________________________________________\n",
      "activation_13 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 16)                272       \n",
      "_________________________________________________________________\n",
      "activation_14 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 3)                 51        \n",
      "_________________________________________________________________\n",
      "activation_15 (Activation)   (None, 3)                 0         \n",
      "=================================================================\n",
      "Total params: 643\n",
      "Trainable params: 643\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "agent = DQNAgent()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EvjpcUbaR3EF"
   },
   "source": [
    "## 3. Train the agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "I7j_aM56tO2_"
   },
   "outputs": [],
   "source": [
    "aggr_ep_rewards = {'ep':[],'avg':[],'min':[],'max':[]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "Xmoxp-PEjN3s",
    "outputId": "d39fc30e-0129-4c0e-a845-907eed7aea69"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|##########| 1000/1000 [36:44<00:00,  1.48s/episodes]\n"
     ]
    }
   ],
   "source": [
    "# Iterate over episodes\n",
    "for episode in tqdm(range(1, EPISODES + 1), ascii=True, unit='episodes'):\n",
    "\n",
    "#     # Update tensorboard step every episode\n",
    "#     agent.tensorboard.step = episode\n",
    "\n",
    "    # Restarting episode - reset episode reward and step number\n",
    "    episode_reward = 0\n",
    "    step = 1\n",
    "\n",
    "    # Reset environment and get initial state\n",
    "    current_state = env.reset()\n",
    "\n",
    "    # Reset flag and start iterating until episode ends\n",
    "    done = False\n",
    "    while not done:\n",
    "\n",
    "        # This part stays mostly the same, the change is to query a model for Q values\n",
    "        if np.random.random() > epsilon:\n",
    "            # Get action from Q table\n",
    "            action = np.argmax(agent.get_qs(current_state))\n",
    "        else:\n",
    "            # Get random action\n",
    "            action = np.random.randint(0, ACTION_SPACE_SIZE)\n",
    "\n",
    "        next_state, reward, done, _ = env.step(action)\n",
    "\n",
    "        # Transform new continous state to new discrete state and count reward\n",
    "        episode_reward += reward\n",
    "\n",
    "#         if SHOW_PREVIEW and not episode % AGGREGATE_STATS_EVERY:\n",
    "#             env.render()\n",
    "\n",
    "        # Every step we update replay memory and train main network\n",
    "        agent.update_replay_memory((current_state, action, reward, next_state, done))\n",
    "        agent.train(done, step)\n",
    "\n",
    "        current_state = next_state\n",
    "        step += 1\n",
    "\n",
    "    # Append episode reward to a list and log stats (every given number of episodes)\n",
    "    ep_rewards.append(episode_reward)\n",
    "    if not episode % AGGREGATE_STATS_EVERY or episode == 1:\n",
    "        average_reward = sum(ep_rewards[-AGGREGATE_STATS_EVERY:])/len(ep_rewards[-AGGREGATE_STATS_EVERY:])\n",
    "        min_reward = min(ep_rewards[-AGGREGATE_STATS_EVERY:])\n",
    "        max_reward = max(ep_rewards[-AGGREGATE_STATS_EVERY:])\n",
    "        \n",
    "        # rewards recording\n",
    "        aggr_ep_rewards['ep'].append(episode)\n",
    "        aggr_ep_rewards['avg'].append(average_reward)\n",
    "        aggr_ep_rewards['min'].append(min_reward)\n",
    "        aggr_ep_rewards['max'].append(max_reward)\n",
    "        \n",
    "#         agent.tensorboard.update_stats(reward_avg=average_reward, reward_min=min_reward, reward_max=max_reward, epsilon=epsilon)\n",
    "\n",
    "        # Save model, but only when min reward is greater or equal a set value\n",
    "#         if min_reward >= MIN_REWARD:\n",
    "#             agent.model.save(f'models/{MODEL_NAME}__{max_reward:_>7.2f}max_{average_reward:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model')\n",
    "\n",
    "    # Decay epsilon\n",
    "    if epsilon > MIN_EPSILON:\n",
    "        epsilon *= EPSILON_DECAY\n",
    "        epsilon = max(MIN_EPSILON, epsilon)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "QQ22Ry_q4mQS"
   },
   "source": [
    "## Plot episodes vs rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 300
    },
    "colab_type": "code",
    "id": "G1TtpQaZVJDA",
    "outputId": "5119d0e5-ca00-4d78-d9d6-887c5c7d7904"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Rewards')"
      ]
     },
     "execution_count": 30,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEKCAYAAADTgGjXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xd40+XawPHv070pLbSMFsqGtixp\nmR4ZshwsRYWjgltfxYEHBy5wDzxwcBwQVxE4AsLhsESWIjKKlE0HMks6aEuheyfP+0dSLFIgbZP+\nkvT5XFeuJr95p5TcebaQUqIoiqIoteGkdQCKoiiK/VJJRFEURak1lUQURVGUWlNJRFEURak1lUQU\nRVGUWlNJRFEURak1lUQURVGUWlNJRFEURak1lUQURVGUWnPROgBra9KkiQwLC9M6DEVRFLuxb9++\n81LKpuYc6/BJJCwsjLi4OK3DUBRFsRtCiGRzj7W56iwhRA8hRKwQ4qAQIk4I0du0XQghPhFCnBBC\nHBZC3KB1rIqiKA2dzSUR4CPgTSllD+AN02uAW4AOpsdjwDxtwlMURVEq2WISkYCf6XkjIM30fAzw\nnTSKBfyFEM21CFBRFEUxssU2keeAjUKIjzEmuf6m7S0BXZXjUkzb0v96ASHEYxhLK7Rq1eqKG5SX\nl5OSkkJJSYllI7cRHh4ehISE4OrqqnUoiqI4OE2SiBBiC9Csml2vAjcDU6WUK4UQdwNfA0Nrcn0p\n5QJgAUBUVNQVC6akpKTg6+tLWFgYQogax2/LpJRkZ2eTkpJCmzZttA5HURQHp0kSkVJeNSkIIb4D\nnjW9/AH4yvQ8FQitcmiIaVuNlZSUOGQCARBCEBgYSFZWltahKIrSANhim0gaMND0fAhw3PR8DTDJ\n1EurL5ArpbyiKstcjphAKjnye1MUxbbYYpvIo8BcIYQLUIKpbQP4EbgVOAEUAQ9qE56iKIptqTBU\ncL74PBlFGZwrPEdGYQblhnIe7vqw1e9tc0lESrkD6FXNdgk8Vf8RKYqiaKcyQZwrPPdnkijKIKMw\ng3NFxoRxvvg8eqm/7Lwmnk0aZhJRFEVp6Iorink39l1i02PJKs7CIA2X7fdw9qCZdzOCvYPp07yP\n8blX8GU//dz8rnJ1y1JJRCNjx45Fp9NRUlLCs88+i8Fg4OTJk8yaNQuAmJgY4uLi+Oyzz3j77bdZ\nvHgxTZs2JTQ0lF69ejFt2jSN34FtqDBUcK7wXK3P93XzpZF7IwtGZJ6SihLcnd1V+5VyhYslF5ny\n8xSOZB1hZJuRhPqGVpsgbOVvp8EnkTfXxpOQlmfRa4a38GPGqIhrHvPNN98QEBBAcXEx0dHRbN26\nlQEDBlxKIsuWLePVV19l7969rFy5kkOHDlFeXs4NN9xAr15X1PY1WK/tfI31p9bX+nwX4cLM/jMZ\n036MBaO6trhzcTz7y7P0ad6Hjwd+jJOwxf4tihZ0+Tqe3PIkaQVpzB40m6GtazS6QRMNPolo5ZNP\nPmHVqlUA6HQ6Tp8+Tdu2bYmNjaVDhw4kJSUxYMAA5s6dy5gxY/Dw8MDDw4NRo0ZZJZ5SfSm/6n5l\naOuhdvOhduLiCX489SO3tb2Nfs371eoaa0+t5bWdr5FZlMkjXR+x+re7jWc2Mv236fi4+rA5eTP/\nPvhvpvScYtV7KvYhPjueJ7c8SYWhgq9GfEXPoJ5ah2SWBp9ErldisIZt27axZcsWdu/ejZeXF4MG\nDaKkpIQJEyawfPlyOnfuzLhx4+q1uLr6xGrejn2bfw78J8PDhtfbfeti/uH5eLp48nL0y/h7+Nfq\nGre2uZXXd73OJwc+4VzhOab3mY6Lk3X+WyxKWMSsvbPo3rQ7nw75lDn75/DF4S9o37g9I8NGWuWe\nin3YkbqD57c9T2P3xswbMY+2/m21Dsls9vGV08Hk5ubSuHFjvLy8SEpKIjY2FoBx48axevVqvv/+\neyZMmADAgAEDWLt2LSUlJRQUFLBu3TqrxBSbboxhYfxCjB3hbNvxi8fZdGYT93a5t9YJBMDV2ZX3\nbnyPhyMfZvkfy5m6bSrFFcUWjBQM0sDHez/mo70fMaTVEL4c/iX+Hv682udVejTtwes7XicxO9Gi\n91RqxiANnC8+T/z5eLYmb2VJ4hJmx83mxe0v8vTPT7MzdafV7r3q+CqmbJ1Ca7/WLL51sV0lEFAl\nEU2MHDmS+fPn06VLFzp16kTfvn0BaNy4MV26dCEhIYHevXsDEB0dzejRo+nWrRvBwcF07dqVRo0s\n2xCsN+jZk76HRu6NOHz+MAezDtp8UfqLw1/g5erFpPBJdb6Wk3DiuV7PEewdzPt73ueRTY/w2ZDP\naOzRuM7XLtOX8dqO19hwZgMTO0/kpeiXcHZyBsDN2Y05g+cwYd0EnvnlGb6/7XuaeDap8z2VK1V2\nkb2se2zlc9OjwlBx2TluTm4EewdTqi/liS1P0L9Ff57v9TydAjpZJCYpJV8c/oLPD35Ov+b9mD1o\nNj5uPha5dn0S9vCtsy6ioqLkXxelSkxMpEuXLhpFVHMFBQX4+PhQVFTETTfdxIIFC7jhhmsvp1KT\n9xh/Pp4J6ycws99M5uyfQ1RwFP8a/C9LhG4Vxy8e5841d/JI10d45oZnLHrtrclbeem3l2jm3Yx5\nN88j1C/0+iddRV5ZHs/+/CxxGXFM7TWVByMerLaKMiE7gckbJtM5oDNfj/gaN2e3urwFpQopJW/F\nvsWKP1Zctr0yQfy111OwVzDB3sbnjd0bI4SgTF/GsmPLmH9oPvll+YxtP5YpPacQ5BVU67gqDBW8\nE/sOK4+vZHS70czsNxNXZ9uZMFUIsU9KGWXWwVJKh3706tVL/lVCQsIV22zZxIkTZffu3WWnTp3k\ne++9Z9Y5NXmPXx7+UkbGRMqsoiw5d99c2TWmqzyTe6a24VrdP7b9Q/ZZ0kdeLL5olesfyDggB3w/\nQN609CZ5NOtora6RXpAux/5vrOzxXQ+57uS66x6/4fQGGRkTKd/Y+YY0GAy1uqdypbn75srImEj5\n1q635M/JP8uE8wkyuzi7Vr/jnJIcOev3WbLndz1l9OJo+dmBz2RhWWGNr1NYViif2vKUjIyJlHP3\nzbXJf28gTpr5Gav5h7y1H46QRGqjJu/x4Y0Py3Grx0kppcwqypI9v+sp3979trVCq5M/Lvwhu8Z0\nlXP3zbXqfU7mnJQjVoyQ0Yuj5Xbd9hqde+zCMTlk+RDZd0lfuTttt9nnfbL/ExkZEykXJyyuabhK\nNRYnLJaRMZFyxs4ZFv2gPpt3Vk7bNk1GxkTKgUsHyh+O/SDL9eVmnXu+6LycuG6i7Lawm1yauNRi\nMVlaTZKIalhv4EoqSjiQcYC+zY3tMk08m3B729tZfWI1OSU5Gkd3pfmH5uPl6sXkiMlWvU/bRm1Z\nfOtiwvzCePrnp1l1fJVZ5/2e/juTN0wGCTEjYy79Xs3xVI+nGBw6mFl7Z7E7bXdtQ1eAn878xIe/\nf8ig0EG81vc1i/Z0DPUNZdbAWSy+dTGt/Frx5u43Gb9mPNtTthu/mV/F2byz3L/hfv64+AdzBs3h\nns73WCwmLakk0sAdzDpImaHssg+7SeGTKNGXsOzYMg0ju9Lxi8fZlGzskVUfo8ybeDbh25Hf0qd5\nH97Y9QbzDs675ofEhtMbeGLLEwR7BbP41sU1boB1Ek68/7f3adOoDdN+ncbZvLN1fQsN0u/pv/PK\nb6/QI6gHs26aZbUu292bdmfhyIXMGTSHckM5T219ikc3P0rShaQrjj2SdYT7N9xPflk+Xw3/iiGt\nhlglJi2o3lkNXGxaLC7ChajgP9vQ2jduz40tb+T7pO95IPIB3J3dNYzwT/MPzcfb1dsiPbLM5e3q\nzWc3f8bMXTP596F/k1GUwWt9X7vsg0lKyXcJ3/Fx3Mf0Cu7F3MFza53kvF29+WTIJ0xcP5Gnf36a\nJbcusViPHV2+jrn757IlecsVczGZy8fNh0+HfEqvYNucNeHYhWM8+8uztPJtxadDPsXDxcOq9xNC\nMLT1UAaGDGT5H8uZd2ged6+9m1HtRvF0z6dp5t2M7SnbmfbrNAI8Apg/dD5hjcKsGlN9U0mkgYtN\nj6Vb0254uXpdtn1yxGQe3fQo60+t544Od2gU3Z/+uPgHm5I38Vi3x+p9ritXJ1feGfAOwV7BfHnk\nS7KKs5h10yy8XL3QG/R8HPcxixMXM7z1cN7723t1TrqhvqHMHjibxzY/xku/vcQngz+51C24NnJL\nc/ni8Bd8n/Q9rk6u3N3p7lpPzrfu1Dre2PkGK0evtPoHdE2l5KfwxJYn8Hb1Zv6w+fX6d+Lq7Mq9\nXe5lVLtRfHX4KxYnLmbTmU3c3Ppmfjr9E50COvH5zZ87ZhducxtP7PVhrw3rq1evlu+//36tzzfn\nPeaU5MiuMV3lvw/++4p9BoNBjl8zXo5eNVrqDfpax2EpU3+ZKvsu6StzSnI0jWNZ0jLZbWE3OWHt\nBJmWnyan/jJVRsZEyg/2fGDx39P3id/LyJhIOSduTq3OL60olTFHY2S///ST3RZ2kzN2zpCZhZl1\nimlP2h4ZGRMp/xn3zzpdx9Kyi7Plbf+9Tfb/T3954uIJrcORKfkp8sVfX5SRMZHy8c2P16oXl5ZQ\nvbPsP4nUlTnvcdOZTTIyJlLuz9hf7f41J9bIyJhI+avuV0uHVyPHLhyTkTGR8tP9n2oaR6Wfk3+W\nUYuiZI+FPWRkTKRceHShVe5jMBjkm7velJExkWZ1E6563obTG+SIFSMufYgdu3DMYnG9sfMN2X1h\ndxl/Pt5i16yLwrJCOWHtBNlrUa+r/i1rJb0gXVboK7QOo8ZqkkRUw7oGzpw5Q+fOnXnggQfo2LEj\n9957L1u2bGHAgAF06NCB33//nZiYGKZMMU7M98ADD/DMM8/Qv39/2rZty4oVK65zB/PEpsXi5eJF\nZJPIavePbDOSIK8gvov/ziL3q635h+bj4+rD/eH3axpHpcGtBvPViK/oHNCZWQNnMSnCOm00Qgim\n955Or+BezNg1g/jz8dc950DmAe7bcB8v/PoC3q7efDH0C+YPnU/Hxh0tFtfzvZ6nsUdjZuyaQbmh\n3GLXrY1yQznPb3uehAsJzLppls3NtNDMu1mdqiLtgWoT2fAynDti2Ws26wq3fHDNQ06cOMEPP/zA\nN998Q3R0NP/5z3/YsWMHa9as4b333mPs2LGXHZ+ens6OHTtISkpi9OjRjB8/vs5hxqbHEt0sGlen\n6kfKujq5cl+X+5i9bzaJ2Yl0Caz/Uf5/XPyDzcmbebzb45qs+3E13Zt25/vbv7f6fVydXZk9aDYT\n103kmV+eYeltS2nq1fSK487mneVf+//F5uTNBHkG8Vb/txjdbrRVPsAauTfi1T6vMnXbVL6L/65e\nVs+rjpSSGTtnsDNtJzP7zWRwq8GaxNHQqZKIRtq0aUPXrl1xcnIiIiKCm2++GSEEXbt25cyZM1cc\nP3bsWJycnAgPDycjI6PO908rSONs/tnrjmO4s+OdeLl4sTBhYZ3vWRu2VgrRQoBHAJ8M+YT8snye\n++U5SvWll/ZdLLnIB79/wJj/jWFH6g6e6vEUa8etZVyHcVb9Bjy09VCGthrKvEPzSM5Lttp9rmXO\n/jmsPbWWp3o8xZ0d79QkBkWVRK5bYrAWd/c/e/A4OTldeu3k5ERFRcU1jzdWWdbNnvQ9ANdNIn5u\nftzZ8U6+T/ye5254jmbezep8b3Mdu3CMzcmbeaL7EzZVCtFCp4BOvHvjuzy/7Xne2v0Wb/R7g/8k\n/ocvD39JYUUhd3S4g6d6PFWvvX9e6fMKY1aPYeaumXw94ut6XYdmUcIivj36Lfd0uofHuz1eb/dV\nrqRKIg3U7vTdNPFsQjv/dtc99r4u9yGRLElcUg+R/emLw1/g4+rDfV3uq9f72qphrYfxZPcnWXNy\nDUN/GMrsfbPpEdSDlaNWMqPfjHrvPtrUqynToqYRlxHHyuMr6+2+P576kY/2fsTQVkOZ3nu6zSwT\n21CpJNIAGaSBPel76Nu8r1n/AVv4tGBY62Gs+GMFBWUF9RDhn6WQ+8Lva/ClkKoe7/44o9uNJtQ3\nlAXDFvDvof+mfeP2msUzrv04+jTrw+y42WQU1r2a9Xp2pe3i1Z2v0iu4Fx/c9IHDN1rbBXO7cdnr\nQ3XxvVJSdpKMjImU/zv+P7OvdzTrqIyMiZQxR2MsEd51Tf1lquy3pJ/m40KU60vOTZa9FvWST299\n2qoz0h49f1T2Xtxbjls9TuaW5lrtPorq4qtcR+Uqhn2a9zH7nIgmEfQK7sWSxCVW79apSiH2pZVf\nK57q8RS/6H5hU/Imq9xDl6fjyS1P0si9EfOHzq/1iHvF8jRJIkKIu4QQ8UIIgxAi6i/7pgshTggh\njgkhRlTZPtK07YQQ4uX6j9pxxKbHEuYXVuNG8gciHiC9MJ3NZzZbKTKj+Yfm4+vqy33hqi3EXtwf\nfj/hgeG8t+c9cktzLXrtA5kHmPzTZAzSwPxh8+u0GJRieVqVRI4CdwDbq24UQoQDE4AIYCTwbyGE\nsxDCGfgcuAUIByaajlVqqFxfzr6MfTWaorzSTSE3EeYXxsIE663DfuzCMbac3cJ94fepb5t2xMXJ\nhTf7v0luaS6z9s6yyDWllCxOWMxDPz2Eh4sHX4/4mraN7Gv98YZAkyQipUyUUh6rZtcYYKmUslRK\neRo4AfQ2PU5IKU9JKcuApaZjlRo6fP4wxRXF9G1R8yTiJJyYFDGJhOwE4jLirn9CLahSiP3qHNCZ\nByMfZPXJ1exK21WnaxWVF/Hi9hf5cO+H/C3kbyy9falFR90rlmNrbSItAV2V1ymmbVfbrtRQbHos\nTsKJ6GbRtTp/VNtRBHgEsDDe8oMPky4kseXsFu4Pv1+VQuzUE92fIMwvjLd2v0VReVGtrnEq9xQT\n109kU/ImnrvhOf41+F/q78GGWS2JCCG2CCGOVvOweglCCPGYECJOCBGXlZVl7dvZldi0WCIDI2v9\nn9LDxYN7Ot3Drym/cir3lEVjqyyF3Bt+r0Wvq9Qfd2d3ZvSbQWpBKp8d/KzG5288s5GJ6yaSU5rD\ngmELeLjrw/U6iNGWnD5fiO5C7RJxfbLav46UcqiUMrKax+prnJYKhFZ5HWLadrXtV7v3AilllJQy\nqmnTK+cZaqgKygo4cv5IjXplVeeeTvfg7uxu0YkZky4ksfXsVlUKcQBRzaK4q+NdLElcwpEs8+al\nKzeU89Hej5j26zTaN27PstuX1fnv1J5l5JUw5rMdDPnnNt7/MZH8Em0nurwWW0vxa4AJQgh3IUQb\noAPwO7AX6CCEaCOEcMPY+L5GwzjtUlxGHHqpp1+LfnW6TqBnIKPajWLtybVkF2dbJDZVCnEsU3tN\npYlHE97Y9Qbl+mt/AGYVZfHIxkdYlLCIv3f+OzEjYup1eh1bI6Vk+n+PUKY3cFvX5iz47RSDP97G\nsr1n0Rus06GlLrTq4jtOCJEC9APWCyE2Akgp44HlQALwE/CUlFIvpawApgAbgURguelYu2TOVPC/\n//47/fr1o2fPnvTv359jx4z9EObMmcNDDz0EwJEjR4iMjKSoyLwib2x6LB7OHnRv2r3O72FS+CTK\nDGUsPba0ztdSpRDH4+vmy2t9X+NEzgm+Pvr1VY+LOxfH3evuJvFCIh/87QOm95mOq3P1s0o3FP/d\nn8rPSZm8OKIz/5rQk9VPDaB1oDcvrTzCmM93sPfMBa1DvIywVldNWxEVFSXj4i7vSZSYmEiXLsZp\nzT/8/UOSLiRZ9J6dAzrzUu+Xrrr/zJkztG/fngMHDhAREUF0dDTdu3fn66+/Zs2aNXz77bd89913\neHl54eLiwpYtW5g3bx4rV67EYDAwaNAgpk6dyrvvvsvcuXMZMGDAFfeo+h4rjf3fWIK9g/li2BcW\neZ9Pb32aQ1mH2DR+U62WSi3Tl7E7bTdfHP6CM3ln+OnOn1QScTDTfp3Gz2d/ZsWoFbT1/7N7rpTG\ndenn7JtjXA540Gw6NO6gYaS2ISOvhGGzf6VzMz+WPtYXJyfjtERSStYeTuf9HxNJzy1hVPcWvHxL\nZ1r6e1olDiHEPill1PWPVLP4aqZyKnig2qngc3NzmTx5MsePH0cIQXm5sUrAycmJmJgYunXrxuOP\nP15tAqlOZlEmJ3NPMqa95fo1TIqYxEMbH2LNyTXc3elus86pTBwbz2xkm24b+eX5+Lr58mL0iyqB\nOKCXe7/M7rTdzNw9k5iRMTgJJwrLC3l95+tsTt7M0FZDeXvA2/i4+WgdquaqVmN9NL7bpQQCxgXK\nRndvwbAuwcz/9SRfbD/J5oRzPH5TO54Y2A5PN+3mEGvwSeRaJQZrut5U8K+//jqDBw9m1apVnDlz\nhkGDBl06/vjx4/j4+JCWlmb2/cyd+r0mooKjiAiMYFHCIsZ3HH/VXjRl+jJ2pe1i05lN/KL7hYLy\nAvzc/Li59c0Mbz2cvs37NvgqDEfVxLMJL0a/yGs7X2PZsWX0btab5355Dl2+jmlR05gUPknNwmtS\nWY31xu3hhDXxrvYYTzdnpg7ryN3RoXywIYm5W4+zPE7Hy7d0ZnT3Fpr8Lht8ErFVubm5tGxpHAoT\nExNz2fZnnnmG7du3M2XKFFasWGHWKoex6bH4u/vTKaCTxWIUQjA5YjIvbn+RX3W/XrayXKm+lF2p\nu9iUvIltum2XEsfQ1kMZETaCPs36qMTRQIxuN5r1p9YzZ98cALxcvPhy+Je1HqvkiDLySnhzbTy9\nwwJ4oH/YdY9v6e/JpxN7Mqlfa95cG8+zSw+yaHcyb4wKp1uIv/UDrkIlERv14osvMnnyZN555x1u\nu+22S9unTp3KU089RceOHfn6668ZPHgwN910E0FBV59PSEpJbHosfZr3sXif+2Gth9Hcuzkx8TH0\nb9mfXam72JhsrKoqLC/Ez82PYa2HMTxsOH2a97nqUryK4xJC8Ea/N7hr7V10aNyBjwd+rOa/quJa\n1VjXEx0WwJqnbmTFvhQ+2niMMZ/vZPwNIbwwshNBvjVvp6yNBt+w7qiqvsdTuacY878xzOg3g/Ed\n6742+199F/8ds+Jm4eXiRVFFEY3cGzEkdAgjwkbQu3lvlTgUAPLL8vF29W6wgwevZuW+FP7xwyHe\nuD2ch25sU+vr5JeU89kvJ/hmx2ncnJ2YMqQDD9/YBjeXmv++VcO6cpnYNOPU75ZsD6nqzo53sj11\nOy19WjK89XCVOJRq+br5ah2CzalpNda1+Hq4Mv2WLkyMbsW7Pyaycn8Kj/yt9knJXCqJNACx6bGE\n+IQQ4htilet7u3rz1fCvrHJtRXFUdanGupawJt58OSmK3OJyXJ2tX+prsOVKR67Gq/reKgwV7D23\nt0FPIaEotmhllUGFV+uNVReNPOunNqBBJhEPDw+ys7MdMpFIKcnOzsbDw9ioFp8dT0F5Qa2mflcU\nxTrO5VquGktrDbI6KyQkhJSUFBx1hl8PDw9CQoxVV5XjQ/o0UyURRbEFUkpeWXWEcgtXY2mlQSYR\nV1dX2rSxfoOTLYhNj6VLQBcaezTWOhRFUfizGmvGqKsPKrQnDbI6q6EoKi/iYOZBq/XKUhSlZqpW\nY03uF6Z1OBahkogDO5B5gHJDuUoiimIDHK0aq5JKIg4sNj0WVydXegb31DoURWnwKquxXhppnd5Y\nWlFJxIHFpsfSM6gnni7WmS5aURTzOGI1ViWVRBzUhZILJF1IUlVZiqIx46DCww5XjVVJJREH9Xv6\n7wBqkKGiaGzl/lR+OZblcNVYlVQScVCx6bH4uvoSHhiudSiK0mA5cjVWpQY5TqQhiE2PJbpZNC5O\n6p9YUepbZn4JCWl5fPHrKYetxqqkPmEckC5fR2pBKpMjJmsdiqI4NINBcia7kPi0PBLS80gw/czK\nLwVACHh3bFeHrMaqpJKIA4pNt+7U74rSEJWU6zl2Lp+E9Dzi03JJSMsj6Vw+RWV6AFycBB2CfRnY\nsSnhzf0Ib+FHl+Z+9TYRolZUEnFAsWmxBHsFE+YXpnUoimLXsgtK+WBDEgd1OZzMKsBgmrPV192F\nLi38uDsqlPAWfkS08KN9kA/uLs7aBqwBlUQcjEEa2HNuD4NCBiGEY9bBKkp9eWtdAj8eSeemDk0Z\nGdmMiBZ+hDdvREhjT4dt46gplUQcTNKFJHJLc9XU74pSR7tPZrP6YBrPDGnP88M7aR2OzVJdfB1M\nZXuImvpdUWqvXG/gjdVHCQ3w5MnB7bUJIusPyE3V5t41oEkSEULcJYSIF0IYhBBRVbYPE0LsE0Ic\nMf0cUmVfL9P2E0KIT4Sqq6nWnvQ9tPdvT1OvplqHoih265sdpzmeWcDMURF4uGrUzrF8Evz4gjb3\nrgGtSiJHgTuA7X/Zfh4YJaXsCkwGFlXZNw94FOhgeoyshzjtSqm+lP0Z+1WvLEWpg/TcYuZuPc7Q\nLkHc3CVYmyAMBrhwCtIPaXP/GtAkiUgpE6WUx6rZfkBKmWZ6GQ94CiHchRDNAT8pZaw0rmn7HTC2\nHkO2C4cyD1GiL1FJRFHq4J11iegNkhmjIrQLojAT9KWQlwLFOdrFYQZbbhO5E9gvpSwFWgIpVfal\nmLYpVWxO3oyzcKZXcC+tQ1EUu7T9jyzWH0lnyuD2hAZ4aRdIju7P55mJ2sVhBqv1zhJCbAGaVbPr\nVSnl6uucGwF8CAyv5b0fAx4DaNWqVW0uYXfis+NZ/sdyxrUfh4+bj9bhKIrdKa3QM2NNPGGBXjw2\nsK22weQk//k8Mx5a99MuluuwWhKRUg6tzXlCiBBgFTBJSnnStDkVCKlyWIhp29XuvQBYABAVFSVr\nE4c9KTeUM2PnDAI9Ank+6nmtw1EUu/TVb6c5fb6QhQ/11n7QYM5Z409Xb8iI1zaW67Cp6iwhhD+w\nHnhZSrmzcruUMh3IE0L0NfXKmgRcszTTkCyMX8ixi8d4te+r+Ln5aR2Ootgd3YUiPv35OLdENmNg\nRxvo2ZirA88AaN4dMhK0juaatOriO04IkQL0A9YLITaadk0B2gNvCCEOmh5Bpn1PAl8BJ4CTwIb6\njtsWnc49zbyD8xjWehg3t7pZ63AUxS69tS4BJyF4/XYbWToh5yz4t4LgcMhMAGm7FSqajFiXUq7C\nWGX11+3vAO9c5Zw4INLKodmxk2yBAAAgAElEQVQVgzQwc9dM3F3ceaXPK1qHoyh26eekDDYnZPDy\nLZ1p4W8jS0nn6KBpRwgKh9I8Y8nE3zbbd22qOkupmRV/rGB/5n5eiHqBJp5NtA5HUexOSbmxMb19\nkA8PDWijdThGUppKIq0h2NTN2IartFQSsVPnCs8xe99s+jTvw9j2asiMotTGvG0n0V0o5q0xEbi5\n2MjHYeF5qCg2ljyCuhi3Zdpu47qN/NaUmpBS8m7su+gNemb0m6Fm61WUWkjOLmTerycZ3b0F/dvZ\nUEk+19Qzq1EoeDSCRq1USUSxrI3JG9mWso0pPacQ6huqdTiKYneklMxYE4+bsxOv3tZF63AuV9m9\nt7INJDjcprv5qiRiZ3JKcnh/z/tEBEZwb5d7tQ5HUezSpoQMth3L4rmhHQj289A6nMtdSiKmL4hB\n4ZB9HCrKtIvpGlQSsTOz4maRV5rHm/3fxMVJLQejKDVVVFbBW2sT6NzMlwf6h2kdzpVydMZqLI9G\nxtfBEWCogPN/aBvXVagkYkd2pe5izck1PBj5IJ0C1CI5ilIbn/18gtScYt4eG4mLsw1+BFaOEalU\n2UMr0zbbRWzwN6hUp6i8iLdi3yLML4zHuz+udTiKYpdOZBbw5W+nuPOGEKLDArQOp3q5OmNjeqXA\n9uDkChlHtYvpGlQSsROfHviU1IJUZvafibuzu9bhKIrdkVIyc008Hq7OvHxLZ63Dqd6lMSJVkoiz\nKzTtbLM9tFQSsQOHsw6zJHEJ93S6R03zrii1tP5IOjtOnOeFEZ1o6mujX8SKL0JZwZWj0yunP7FB\nKonYuHJ9OTN2zSDIK4jnbnhO63AUxS4VlFbw9roEIlv6cW+f1lqHc3V/7ZlVKSgc8lKNScbGqCRi\n474++jUnck7wet/X1TohilJLn2w9TkZeKW+PicTZyYYH5/51jEglG57+xKwkIoRoJ4RwNz0fJIR4\nxjRtu2JFp3JOseDwAm4Ju4WBoQO1DkdR7NKxc/l8s+M0E6JD6dmqsdbhXFtOldHqVdlwDy1zSyIr\nAb0Qoj3GxZ5Cgf9YLSoFgzQwY9cMvFy9eKn3S1qHoyh2KS2nmIcX7sXP05UXR9poY3pVuTpw8wXP\nvyQ73+bg4W+TI9fNTSIGKWUFMA74VEr5AtDcemEpS5OWcjDrIC9Fv0SgZ6DW4SiK3cnIK+HvX8aS\nW1ROzIPRBHi7aR3S9VX2zPrrfHhCGEsjdpxEyoUQE4HJwDrTNlfrhKSkF6Qzd/9c+rfoz+1tb9c6\nHEWxO1n5pfz9y1iy8kuJeag33ULspPY9R3dlo3qloHDITLS5BarMTSIPYlyF8F0p5WkhRBtgkfXC\nariklLwV+xYSyRv93lAz9CpKDV0oLOO+r/aQllPCNw9E06u1jbeDVPXXMSJVBUdAWf6f7SY2wqzJ\nl6SUCcAzVV6fBj60VlAN2frT69mRuoOXol+ipU9LrcNRFLuSW1TO/V/v4XR2Id8+EE2ftnZUFVyc\nA6W5104iYGxcb2w73ZSvmUSEEEeAq5adpJTdLB5RA7cwfiGdAzozsfNErUNRFLuSX1LOpG9/53hG\nAV9M6sWA9ja0Rog5cnXGn3/tmVWpcoGqjKPQ6Zb6ickM1yuJVFbIP2X6WVmFdR/XSC5K7UgpSc5L\n5s4Od+Ls5Kx1OIpiNwpLK3jg273Ep+Yy775eDO4UpHVINXe1MSKV3H2N+2xsrMg1k4iUMhlACDFM\nStmzyq6XhBD7gZetGVxDk12STXFFMSG+IVqHoih2o7hMz8ML93JQl8OnE3syLDxY65BqJ8dUErla\nEgEIjrS5sSLmNqwLIcSAKi/61+BcxUy6fOMfkVqtUFHMU1Ku57FFcew5fYHZd3fn1q52PPIg5yy4\neoHXNdpxgsLh/HGoKK2/uK7D3FWNHgK+FUKYVkkhx7RNsSCVRBTFfGUVBp5csp/fjp/no/HdGNPD\nzjui5CRXP0akquBwkHrIOgbNbaNJ+rpJRAjhBLSXUnavTCJSylyrR9YA6fJ1CITqlaUo11GuN/D0\n9/v5OSmTd8dFcneUA3zxytVdvVG9UnCk8Wdmgs0kketWSUkpDcCLpue5KoFYjy5fRzPvZrg528HI\nWkXRSIXewNRlB9kYn8HMUeG2PStvTVxrjEilgHbg7G5TI9fNbdfYIoSYJoQIFUIEVD5qe1MhxF1C\niHghhEEIEVXN/lZCiAIhxLQq20YKIY4JIU4IIRyyQT8lP0VVZSnKNegNkhdXHGbd4XReubUzDwxo\no3VIllGab5zm/XpJxNkFmna0yyRyD8ZuvtuBfaZHXB3uexS4w3S96swGNlS+EEI4A58DtwDhwEQh\nRHgd7m+TdPk6lUQU5SoMBskr/z3Cfw+k8o9hHXnspnZah2Q5l3pmmfH/PyjCpnpomTti3aLpXkqZ\nCFQ7pYcQYixwGiissrk3cEJKecp0zFJgDGA7v8k6Kiwv5ELJBdW9V3FYK/elcC6vBG83Z7zdXfD1\ncMHb3fjwqfzp5oK3uzMuzpd/v5VSMmNNPMvidDw9pD1P39xBo3dhJZfGiJhRNRccAYeXQtEF8NJ+\nnXhze2chhIjEWArwqNwmpfzOksEIIXyAl4BhwLQqu1oCuiqvU4A+lry31lLyUwDVM0txTCcyC/jH\nD4fMPt7dxelSYvF2d0EACel5PH5TW54f1tF6gWrleqPVqwo2VcJkJkDYjdaLyUxmJREhxAxgEMYk\n8iPGaqUdwFWTiBBiC9Csml2vSilXX+W0mcAcKWVBXSYeFEI8BjwG0KrVdeoYbYTq3qs4suVxOlyc\nBL++OBgPFycKS/UUlFZQWFZBQWkFBSUVFJYanxeW6i9tLyz9c/s/hnVkypD2jjkpaU4yuHiAjxkj\n7YMqVzmMt58kAowHugMHpJQPCiGCgcXXOkFKObQW8fQBxgshPgL8AYMQogRjG0zVT9cQIPUa916A\ncfEsoqKi7GJ6FpVEFEdVVmFg5b4UhnYJpqW/JwCBaqXny+WcNZZCzEmQvs2Mi1bZSOO6uUmkWEpp\nEEJUCCH8gEwu/1C3CCnl3yqfCyFmAgVSys+EEC5AB9MU9KnABODvlr6/lnT5Ovzd/fF189U6FEWx\nqK2JGWQXlnFPb/UF6aqutY7IXwlhU9OfmNs7K860pvqXGEsF+4Hdtb2pEGKcECIF4xol64UQG691\nvGlVxSnARiARWC6ltI00bCG6fB0hPqpRXXE8S/fqaN7Ig5s6NNU6FNtlzhiRqioXqDIYrBeTmczt\nnfWk6el8IcRPgJ+U8nBtbyqlXAWsus4xM//y+keM7TEOSZevo1sT2xiBqiiWkppTzPbjWTw9pAPO\nTg7YlmEJZYVQdL5mSSQ4HMoKjG0pAdqOlTGrJCKEWCSEeFQI0VlKeaYuCUS5UrmhnHOF51T3XsXh\n/BBnbOu7q5f6276qXGPPTBrVpCRSZYEqjZlbnfUN0Bz4VAhxSgixUgjxrBXjalDSC9LRS71qVFcc\nit4g+SEuhRvbNyE0wEvrcGzX9dYRqc6lBarsJIlIKX8B3gVex9guEgX8nxXjalBUzyzFEe04cZ7U\nnGImRNtHN3vNXEoiNfj/7+4DjcMgU/umYXPHiWwFvDE2pv8GREspM60ZWEOikojiiJbtPUuAtxtD\nw+1wlcH6lHMWnFzBp7phddcQFGET3XzNrc46DJQBkUA3IFII4Wm1qBoYXb4Od2d3mnqp3iuKYzhf\nUMrmhAzu6NkSdxe11PM15Zw1lkKcarjOX3AEZJ+E8hLrxGUmc6uzpkopb8I4aWI28C3GhakUC6js\n3usk1GKRimNYtT+Vcr3knmhVur4uc9YRqU7lAlXnj1k+phowt3fWFCHEMuAAxokPv8E49YliAWr2\nXsWRSClZuvcsvVo3pkOwGjx7XTUdI1Kp6vQnGjJ3xLoHxunZ95kG/ikWIqUktSCVvs37ah2KoljE\nvuSLnMwq5KPxDjRVu7WUl0BBRu2SSEBbm1igytzqrI8BV+B+ACFEU9MUJEodnS8+T3FFsSqJKA5j\n6V4dPu4u3Na1udah2L7KMSK1SSLOLhDUWfOxIuZWZ83AOEX7dNMmV64zAaNiHtUzS3EkeSXlrD+c\nzqjuLfB2N3uliYYrJ9n4szZJBEw9tOwgiQDjgNGYFoqSUqYBqrLTAlQSURzJ2kNpFJfrmaAa1M1T\nk3VEqhMcDgXnoDDbcjHVkLlJpExKKQEJIITwtl5IDYsuX4eTcKKlT0utQ1GUOlu2V0fnZr50C2mk\ndSj2IecsOLmAby2r/oIqF6jSrl3E3CSyXAjxBeAvhHgU2AJ8Zb2wGg5dvo5mXs1wdXbVOhRFqZP4\ntFwOp+QyITrUMReOsoacs+DX0ti+URvBkcafGlZpmTuL78dCiGFAHtAJeENKudmqkTUQKfkpqipL\ncQjL9+pwc3FibE9VqjZbjq727SFgXAnRK9AuSiJIKTdLKV+QUk4Dtgoh7rViXA2GLl+nZu9V7F5J\nuZ5VB1K5JbIZ/l5uWodjP2o7RqSSEMYqLQ27+V4ziQgh/IQQ04UQnwkhhgujKcAp4O76CdFxFZQV\ncLH0oiqJKHbvp6PnyCupUCPUa6KiDPLTa9+oXik4AjKTNFug6nrVWYuAixgnXnwEeAUQwFgp5UEr\nx+bwVM8spTZKK/S4OjnhZEOLPC3de5bWgV70bROodSj2Iy8FkHUriYAxiZQXQs4Z4wDEena9JNJW\nStkVQAjxFZAOtJJSajvjl4NQSUQxV2ZeCT8nZbIlMYMdJ84T7OfBkkf6ENJY+3U6Tp8vJPbUBV4Y\n0cmmEpvNq806ItWpOv2JDSaR8sonUkq9ECJFJRDLUUlEuRopJQnpeWxNzGRrYgaHUnIBaOnvyR03\nhLDuUBoTFsSy9LG+mieS5XE6nJ0E49XqhTWTYxojUpN1RKoT1BkQxh5aXUbVOayaul4S6S6EyDM9\nF4Cn6bUApJTSz6rROThdvo7G7o3xcfPROhTFBpRW6Nl9MvtS4kjLLUEI6B7iz7ThHbm5SzCdm/ki\nhGBCdCj3fbWHiV/G8v2j2iWScr2BFftSGNwpiGA/D01isFs5Z0E4Gbv41oWbt3GddY16aF0ziUgp\n1UIAVqS69yrnC0r5xVRN9dvx8xSV6fF0debGDk14dmgHBncOIsj3yg/nbiH+LH6kD/eaEsnSx/rR\n0r/+l/j5JSmTrPxSNUK9Ni6NEbHAGLGgcM3GiqjJbTSky9fRI6iH1mEoGlixL4X/7EnmgC4HKaGZ\nnwfjerZkaJdg+rULxMP1+t/fuoX4s8SUSCYs2K1JIlm2V0eQrzuDOqkF1WqstuuIVCc4Ao79COXF\n4Fq/fwNqFSSNlOvLOVd0TpVEGqD9Zy8y7YdDFJbqefbmDqx7+kZ2Tx/Cu+O6MrhzkFkJpFK3EH8W\nP9yHnKJyJi6IJTWn2IqRX+5cbgm/HMvkrqgQXJzVR0mN1XWMSFVB4SANkJVkmevVgPqX10hqQSoG\naVBJpIExGCQz18QT7OfOf5/sz3NDOxLZslGdpgnpHurPoof7cLGojIkLYkmrp0SyYp8Og4S7o9Tf\ncI3pKyAvre6N6pU0nP5EJRGNqJ5ZDdOKfSkcTsll+i1dLDpVeo/KRFJYxoR6SCQGg2RZnI7+7QJp\nHajmY62xvFTj0raWKokEtAEXT03WFtEkiQgh7hJCxAshDEKIqL/s6yaE2G3af0QI4WHa3sv0+oQQ\n4hNh5zO8qSTS8OQWl/PhT0n0at2YMT1aWPz6PUL9WfSIMZFM/NK6iWT3qWx0F4rVCPXastQYkUpO\nztC0E2Qctcz1anLrer+j0VHgDmB71Y1CCBeMi109IaWMAAbx51iVecCjQAfTY2R9BWsNunwdni6e\nNPFsonUoSj35ZOtxLhSV8eboCKvNctsj1J/vHu7NhQJjIknPtU4iWbpXRyNPV0ZENLPK9R1eXdcR\nqU6wNgtUaZJEpJSJUspj1ewaDhyWUh4yHZdtGuTYHPCTUsaa1jX5DhhbjyFbXEpBCi19WqopsxuI\n4xn5LNx1hgnRrYhsad21Nnq2anwpkUxYYPlEcrGwjI1HzzGuZ8sadQJQqsg5CwhoZMEBmsERUJgJ\nhectd00z2FqbSEdACiE2CiH2CyFeNG1vCaRUOS7FtK1aQojHhBBxQoi4rKwsK4Zbe2qMSMMhpeTN\ntQl4uTkzbXjHerlnz1aNWfhwb7ILjI3tlkwkqw6kUqY3qKqsusg5a1yIysXdctesXKCqnmf0tVoS\nEUJsEUIcreYx5hqnuQA3Aveafo4TQtxc03tLKRdIKaOklFFNm9pe/3UppUoiDcimBON8V88P60ig\njwU/NK7jBlOJ5LwpkZzLrfuMRVJKlu3V0T3Uny7N1YQVtZZz1nI9syoFV5lDqx5ZLYlIKYdKKSOr\neay+xmkpwHYp5XkpZRHwI3ADkApULfeFmLbZpaziLEr0JSqJNAAl5XreWZ9Ax2Af7uvbut7vXzWR\nTFiwu86J5KAuh2MZ+WqEel1ZcoxIJZ8g8GpS79Of2Fp11kagqxDCy9TIPhBIkFKmA3lCiL6mXlmT\ngGslI5umemY1HF/9dgrdhWJmjorQbEDeDa0as/AhU4nky7qVSJbt1eHl5syo7pbvXdZgGPTGLr6W\nbFSvpEHjuibTngghxgGfAk2B9UKIg1LKEVLKi0KI2cBeQAI/SinXm057EogBPIENpoddUkmkYUjL\nKebzX05yS2Qz+rfXthder9aNWfhQNJO/2cvIudsJ8nXH1dnJ9BC4Ojvh4uyEW5Xnrs4CN2cnXEzb\nXJ2dWHMojdu7NcfHgmNcGpz8dDBUWL4kAsYkEvetMVE51U+nB03+EqSUq4BVV9m3GGM3379ujwMi\nrRxavdDl63AWzjT3aa51KIoVvb8hCYOUvHJrF61DAaBX6wCWPNKHb3eepqTcQIXBQJleUl5hoFxv\noLBMT4Xe+LxcL00/L3/u4uTEpH5hWr8V+2bpMSJVBYVDRTFcPAOB7Sx//WqorxMa0OXraObdDFcn\nC8zeqdikPaeyWXsojWdv7kBogPYLR1XqHurPvyb01DqMhu3SOiJWKomAsXG9npKIrbWJNAgp+SmE\n+KoFfBxVhd7AjDXxtPT35ImB9fMfWbEjlSURS44RqdTUtEBVPU5/opKIBnT5OtUe4sC+36sj6Vw+\nr97WBU83NRhP+YucZPAOss6U7W5exiVy63H6E5VE6ll+WT45pTkqiTionKIy/rnpGP3aBnJLpJoS\nRKlGrs46VVmVgut3gSqVROqZ6pnl2GZv/oO84nJmjA5XU9oo1bPGGJGqgiPhwikoK7LePapQSaSe\nqSTiuBLT81gcm8z9fVvTuZkaza1Uw2CA3BTLj1avKigckPW2QJVKIvVMJZErJabncSqrQOsw6kRK\n42JTjTxdmTqsfubHUuxQQQboy6xcEqnf6U9UF996lpKfQoBHAN6uaiEfgIy8Eu6ct4uScj139Qrl\nuWEdaN6oftaIzsgr4ZAuh/7tm9R58Nz6I+nsOX2Bd8dF4u/lZqEIFYdzaYyIFafAaRxWrwtUqSRS\nz3T5OtW9t4oPf0qiQi+Z0LsVK+JS+N/BVB4YEMaTA9vTyMs642jOnC/ki+0nWbnPOButp6szt3Vr\nzj3RoUS1blzjtoyisgreW59IeHM/JkRb8RumYv+ssY7IXzk5Q1AXVRJxVLp8HTcE36B1GDbhoC6H\n/+5P5YmB7Xj5ls7838B2zNnyBwu2n+L7PWd5cnB7HugfZrE1K+LTcpm37SQ/HknHxcmJ8VEhDAsP\nZuPRc6w9lMaKfSm0berN3VGh3HFDS4J8Pcy67vxtJ0nLLWHuxJ44O6nGdOUacpKNP63ZJgLGHlrH\nfrLuPUxUEqlHZfoyzhWeU+0hVK6xEU9TX3emDGkPQGiAF7Pv7sGjf2vLrI3H+GBDEjE7z/Dc0A6M\n7xVS6wkMfz99gX9vO8G2Y1l4uznz6N/a8vCNbQjyMyaJwZ2CeP32cNYfSeeHOB0fbEhi1sZjDO4U\nxD3RoQzu1PSq99ZdKGL+9lOM7t6C6LCA2v0ylIYj5yx4BYKblauz+zwB3f8OUoKVewmqJFKPUgtS\nkUiVRIDVB9M4cDaHj8Z3u6I9oktzP755IJo9p7L54KckXv7vEb787RQvjOjMiIhgs6qbpJT8ciyT\nf/9ykrjkiwR4u/GPYR2Z1C+s2moyb3cX7o4K5e6oUE5mFbA8TsfKfalsScygqa87d94Qwt1RIbRt\n6nPZee+sT8BZCKbf2rluvxClYcix8hiRSs26Wv8eJiqJ1CPVM8uoqKyCDzYk0bVlI8bfcPX2oT5t\nA/nv//VnY3wGszYm8cTiffRs5c/LIzvTp21gtedU6A2sP5LOvG0nSTqXT4tGHswYFc6E6FZmjx5v\n19SH6bd0YdrwTvySlMnyuBS+/O0U8389SXRYY+6OCuW2bs3Zn5zDxvgMXhjRqd46Ayh2LuessarJ\ngagkUo9UEjGav+0k5/JK+Pzenjhdpw1BCMHIyGYM7RLEin0p/GvLce5ZEMvgTk15cWTnS6vrlZTr\nWbk/hS9+PcXZC0W0a+rNx3d1Z0yPFrjWshrM1dmJ4RHNGB7RjMy8ElbuT+WHOB0vrDjMzDXxeLq5\n0CrAi4dvbFOr6ysNjJTGhvWOI7SOxKJUEqlHKfkpeLp4EuhR/bfohkB3oYgvTG0IvVqb34bg4uzE\nhN6tGNOjJTG7zjBv2wlu/eQ3xvVoSYdgX77ZeZqs/FK6hzTilVt7MTw8+LoJqiaC/Dz4v0HteGJg\nW+KSL7Jsr45txzJ5a0yExRr+FQdXmAUVJdbt3qsBlUTqUWX33oY8HcYHG5IQAl6+pXZtCJ5uzvzf\noHZM7B3KvG0n+XbXGcoOpDKgfSD/uqcH/dsFWvX3K4QgOixANaIrNWfNdUQ0pJJIPdLl6wjzC9M6\nDM3sOZXN+iPpPDe0Ay3869aG4O/lxvRbu/DwjW3IKymnfZCvhaJUFCu5lEQcqzpbTXtSTwzSQEp+\nSoNtD9EbJG+uTaBFIw8ev8lya2wE+XmoBKLYh0vriDjWZ4BKIvUksyiTMkNZg00iP8TpSEjPY/qt\nao0NpYHK1YGHP3g41uScKonUk4bcMyuvpJxZG48RHdaY27updeWVBsraU8BrRLWJ1JOU/BSgYSaR\nz34+wYWiMmJu792gOxUoDVzOWQhsr3UUFqdKIvVEl6/DWTjTzKdhrXZ3KquAb3ee5q5eIXQNaaR1\nOIqiDSnrb7R6PVNJpJ7o8nU0926Oq5N1Zqa1Ve+uT8TdxZlpIzppHYqiaKfoApQXqiSi1J4uX9fg\nqrJ+/SOLrUmZPD2kvdkz4iqKQ7o0e69KIhYhhLhLCBEvhDAIIaKqbHcVQiwUQhwRQiQKIaZX2TdS\nCHFMCHFCCPGyFnHXRUNLIuV6A2+vS6B1oBcPDAjTOhxF0VZ9rCOiEa1KIkeBO4Dtf9l+F+AupewK\n9AIeF0KECSGcgc+BW4BwYKIQwm5mMcstzSWvLK9BJZElscmcyCzgtdvCcXdRXXqVBs5BR6uDRr2z\npJSJQHU9dSTgLYRwATyBMiAP6A2ckFKeMp23FBgD1M/6j3XU0HpmXSwsY86W49zYvglDuwRpHY6i\naC9HB+5+4OmvdSQWZ2ttIiuAQiAdOAt8LKW8ALQEdFWOSzFtswuVY0QayrK4c7b8QUFpBa/fHq66\n9CoKOOwYEbBiSUQIsQWorj/rq1LK1Vc5rTegB1oAjYHfTNep6b0fAx4DaNVK+3+4hjTQMOlcHotj\nk7m/b2s6NVPTkSgKYEwijR1r9t5KVksiUsqhtTjt78BPUspyIFMIsROIwlgKqfoJHAKkXuPeC4AF\nAFFRUbIWcVhUSkEKgR6BeLl6aR2KVUkpeXtdAr4erjw3tKPW4SiKbahcRyTsRq0jsQpbq846CwwB\nEEJ4A32BJGAv0EEI0UYI4QZMANZoFmUNNZSeWZsTMth5Ipvnh3Wksbeb1uEoim0oyYHSPIetztKq\ni+84IUQK0A9YL4TYaNr1OeAjhIjHmDi+lVIellJWAFOAjUAisFxKGa9F7LXREJJIaYWed39MpEOQ\nD/f2ccz/LIpSKw46BXwlrXpnrQJWVbO9AGM33+rO+RH40cqhWVyZvoyMwgyHTyLf7jxDcnYRix7u\njUstl6NVFIeUY+oTpEoiSm2kFKQgkQ7dMyv2VDafbj3O0C7B/K1DU63DURTbcqkk4pgN6yqJWJmj\njxFZsieZ+77aQ3AjD94cE6F1OIpie3J14OoNno21jsQq1FTwVuao3XvL9QbeWpvAothkBnVqyicT\ne+Ln0bAml1QUs1SOEXHQMVMqiViZLl+Hl4sXAR4BWodiMRcKy3hyyT5iT13g8Zva8uLIzjg7OeZ/\nEEWps5xkh20PAZVErE6XryPEN8RhRm4fO5fPI9/tJSOvlNl3d+eOGxy3rUdRLCJHB6F9tI7CalQS\nsTJdvo62jdpqHYZFbIo/x9RlB/F2d2HZY33p2cox63gVxWJKco3jRBy4JKIa1q3IIA2k5qfafXuI\nlJLPfj7OY4v20S7IhzVTblQJRFHMkeO4U8BXUiURK8osyqTMUGbXSaS4TM8LKw6x7nA6Y3q04MM7\nu+HhqqZ2VxSzVK4j4qDde0ElEauy99l703KKeWxRHPFpebw0sjNPDGzrMG07ilIvHHgdkUoqiViR\nPXfv3Zd8gccX7aekXM9Xk6K4uUuw1iEpiv3JOQsunuDdROtIrEYlESvS5etwES40926udSg1sjxO\nx2urjtLc34PvH+1Dh2A1pbui1ErOWeOcWQ5cgldJxIp0+Tqa+zTHxck+fs0VegPvb0ji6x2nGdA+\nkM//fgP+Xmo2XkWpNQdejKqSfXy62Sl7mr1Xd6GIV1Yd4bfj53mgfxiv3tYFVzWRoqLUTa4OWvTU\nOgqrUknEinT5Oro26ap1GNe0L/kiX+84xU9Hz+HsJHj/jq5M7O3Y35wUpV6c/AWKsh12RcNKKolY\nSW5pLvll+TZZEqnQG9I37xIAAAwJSURBVPgp/hxf7zjNgbM5+Hm48OhNbZncL4wW/p5ah6co9u/Q\nMlj9JASFQ4/7tI7GqlQSsRJb7N6bV1LOst91xOw6Q2pOMa0DvXhzdATje4Xg7a7+FBSlzqSEHXNg\n65sQ9je4ZzF4+msdlVWpTw4rsaXuvboLRXyz8zTL9+ooLNPTu00AM0aFc3OXYDVxoqJYikEPG16E\nvV9B5J0wdh64uGsdldWpJGIll0oiPtqURKSUpvaO02yMP4eTENzerTkP39iWriGNNIlJURxWeTGs\nfASS1kH/p2HoW+DUMDqmqCRiJbp8HU08m+Dl6lWv9y3XG9hw1NjecUiXQyNPVx4f2I7J/cJo1sij\nXmNRlAah6AL85x5I2QsjP4C+/6d1RPVKJRErqY/uvcVles5eKOL0+UKSsws5k13Er8cyScstoU0T\nb94eE8GdvULwclP/zIpiFRfPwOLxxvEgd8VAxFitI6p36tPFSnT5Ovo271vn6xSX6TmT/WeSOHO+\nkDPZhZw5X8S5vJLLjg30diO8hR9vjYlkSOcgnFR7h6JYT9pBWHIX6Eth0v+gdX+tI9KESiJWUFJR\nQmZRZo17ZpWU61l9MJX9yTmcNiWOjLzSy45p4uNG60BvBrRvQligF2FNvAkL9KZVoBeNPNXytIpS\nL05sgeWTjeumT14LQZ21jkgzKolYQWpBKmB+z6yMvBK+232G/+w5y8Wicpr4uNGmiTd/69CUNk28\naR3oRVig8aevWsdcUbR1YAmsfQaadoF7fwA/+5obz9JUErECc7v3HknJ5esdp1h3OB29lAwPD+ah\nAW3o3SZATbmuKLZGSvjtY/j5HWgz0DgGxMNP66g0p5KIFVwriegNks0Jxt5Te89cxMfdhUn9wnig\nfxitAuu3J5eiKGbSV8CP02Dft9DtHhj9GbioyUlBoyQihJgFjALKgJPAg1LKHNO+6cDDgB54Rkq5\n0bR9JDAXcAa+klJ+oEXs5tDl6/B29aax+59LyOaVlLN8r3G0eMrFYkIDPHn99nDujgpRVVSKYsvK\nimDFQ/DHBrhxKtw8w6Gndq8prUoim4HpUsoKIcSHwHTgJSFEODABiABaAFuEEB1N53wODANSgL1C\niDVSygQNYr+uyu69QgiSswuJ2XWGH+JSKCitoHebAF67LZxh4Wq0uKLYHIMBis5DbgrkpUFeKhxe\nBqn74daPofejWkdoczRJIlLKTVVexgLjTc/HAEullKXAaSHECaC3ad8JKeUpACHEUtOxNptEAl1b\n89h3cWxOzMDFSTCqWwseHNBGjRZXFK0YDFCYZUwMlQkiLxVyK1+nQF46GMovP8+9EdyzCLqM0iZu\nG2cLbSIPActMz1tiTCqVUkzbAHR/2d7HmkGNXdCNMmGo1bmprtAz5zQP5GzjPX9X/L3ccDkvYLWF\ng1QUxTzlRdUnCGc38GsBfiEQ2hcatQS/lqZtpudegQ1mCpPasFoSEUJsAZpVs+tVKeX/t3f/sVbX\ndRzHny8hgWQCV8n4KTCZRWZAdw1may7U1JxuxYZkk4qNsdE0VytIN+DPWos0m4MI7QdDSw0RN8GQ\nVs3k12T8lMBJCdOAhrTSOaB3f3w+B75cft37vYd7uN/zemzf3fP9fD/nns/nvO/u+3y+38/5fp7L\ndR4EjgFL6/zaM4AZAMOHl1sbYyB9OR7HSz138Afi5j4DGfHR/j5lZXYx6Nn71MRw+WDoNzQlCF/f\n6JQLlkQi4qZzHZf0NeAOYFJERC7eDxSnNA3NZZyj/EyvvQhYBNDa2hpnq3cuP5/xSpmnmZk1lYaM\n0fJMq+8Cd0bEe4VDK4C7JfWSNBIYDawHNgCjJY2UdCnp4vuKrm63mZmdqlHXRB4FegEv5S/VvRoR\nMyNiu6Tfki6YHwNmRaRzSpK+CawiTfFdEhHbG9N0MzOr0ckzSdXU2toaGzdubHQzzMy6DUmbIqK1\nPXU95cDMzEpzEjEzs9KcRMzMrDQnETMzK81JxMzMSqv87CxJB4G/l3jqlcChOjfnYuc+Nwf3uTl0\nps9XR8TA9lSsfBIpS9LG9k5xqwr3uTm4z82hq/rs01lmZlaak4iZmZXmJHJ2ixrdgAZwn5uD+9wc\nuqTPviZiZmaleSRiZmalOYmcgaRbJe2StEfS7Ea3p14kDZO0VtIOSdsl3Z/LWyS9JGl3/jkgl0vS\nI/l92CJpfGN7UI6kHpJek7Qy74+UtC7366m8vAB5CYKncvk6SSMa2e7OkNRf0tOSXpe0U9LEKsdZ\n0gP5b3qbpGWSelcxzpKWSDogaVuhrMNxlTQt198taVpn2uQk0oakHsDPgNuAMcBUSWMa26q6OQZ8\nOyLGABOAWblvs4E1ETEaWJP3Ib0Ho/M2A3is65tcF/cDOwv7PwAWRMQ1wGFgei6fDhzO5Qtyve7q\nYeDFiPgY8ClS/ysZZ0lDgPuA1oi4jrRcxN1UM85PALe2KetQXCW1AHNJS4x/BphbSzylRIS3wgZM\nBFYV9ucAcxrdrgvU1+eAm4FdwKBcNgjYlR8vBKYW6p+o11020iqYa4DPAysBkb6A1bNtvEnr1UzM\nj3vmemp0H0r0uR/wZtu2VzXOwBDgLaAlx20l8IWqxhkYAWwrG1dgKrCwUH5KvY5uHomcrvYHWbMv\nl1VKHsKPA9YBV0XE2/nQO8BV+XEV3oufkFbR/F/evwJ4NyKO5f1in070Nx8/kut3NyOBg8Dj+TTe\nYkmXUdE4R8R+4EfAP4C3SXHbRPXjXNPRuNY13k4iTUhSX+AZ4FsR8e/isUgfTSoxZU/SHcCBiNjU\n6LZ0sZ7AeOCxiBgH/JeTpziAysV5AHAXKXkOBi7j9FM+TaERcXUSOd1+YFhhf2guqwRJHyIlkKUR\n8Wwu/qekQfn4IOBALu/u78UNwJ2S9gJPkk5pPQz0l1RbGrrYpxP9zcf7Af/qygbXyT5gX0Ssy/tP\nk5JKVeN8E/BmRByMiKPAs6TYVz3ONR2Na13j7SRyug3A6Dyz41LSBboVDW5TXUgS8AtgZ0T8uHBo\nBVCboTGNdK2kVn5vnuUxAThSGDZf9CJiTkQMjYgRpDi+HBH3AGuBybla2/7W3ofJuX63+7QeEe8A\nb0m6NhdNAnZQ0TiTTmNNkPTh/Dde62+l41zQ0biuAm6RNCCP4m7JZeU0+iLRxbgBtwN/A94AHmx0\ne+rYr8+ShrpbgM15u510PngNsBv4A9CS64s0U+0NYCtp9kvD+1Gy7zcCK/PjUcB6YA/wO6BXLu+d\n9/fk46Ma3e5O9HcssDHHejkwoMpxBuYDrwPbgF8DvaoYZ2AZ6brPUdKIc3qZuALfyP3fA3y9M23y\nN9bNzKw0n84yM7PSnETMzKw0JxEzMyvNScTMzEpzEjEzs9KcRMzaQdJxSZsL2znv7ixppqR76/C6\neyVd2dnfY3aheIqvWTtI+k9E9G3A6+4lze8/1NWvbdYeHomYdUIeKfxQ0lZJ6yVdk8vnSfpOfnyf\n0houWyQ9mctaJC3PZa9Kuj6XXyFpdV4bYzHpC2O11/pqfo3NkhYqrZPSQ9ITeR2NrZIeaMDbYE3M\nScSsffq0OZ01pXDsSER8EniUdNfgtmYD4yLiemBmLpsPvJbLvg/8KpfPBf4SEZ8Afg8MB5D0cWAK\ncENEjAWOA/eQvpk+JCKuy214vI59NjuvnuevYmbA+/mf95ksK/xccIbjW4ClkpaTbkEC6RY0XwaI\niJfzCORy4HPAl3L5C5IO5/qTgE8DG9LtoehDutHe88AoST8FXgBWl++iWcd5JGLWeXGWxzVfJN3D\naDwpCZT58CbglxExNm/XRsS8iDhMWrnwj6RRzuISv9usNCcRs86bUvj51+IBSZcAwyJiLfA90m3H\n+wJ/Jp2OQtKNwKFIa7v8CfhKLr+NdONESDfYmyzpI/lYi6Sr88ytSyLiGeAhUqIy6zI+nWXWPn0k\nbS7svxgRtWm+AyRtAT4gLT1a1AP4jaR+pNHEIxHxrqR5wJL8vPc4eSvv+cAySduBV0i3OScidkh6\nCFidE9NRYBbwPmkFw9oHwjn167LZ+XmKr1kneAquNTufzjIzs9I8EjEzs9I8EjEzs9KcRMzMrDQn\nETMzK81JxMzMSnMSMTOz0pxEzMystP8Dyny2pB56Sd4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(aggr_ep_rewards['ep'], aggr_ep_rewards['avg'], label = 'avg')\n",
    "plt.plot(aggr_ep_rewards['ep'], aggr_ep_rewards['min'], label = 'min')\n",
    "plt.plot(aggr_ep_rewards['ep'], aggr_ep_rewards['max'], label = 'max')\n",
    "plt.legend(loc='upper left')\n",
    "plt.xlabel('Episodes')\n",
    "plt.ylabel('Rewards')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nwD41YstHyGx"
   },
   "source": [
    "### Save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YU3N0zE_HxQb"
   },
   "outputs": [],
   "source": [
    "agent.model_prediction.save('dqn_1.model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "9ILbxFFfIckN"
   },
   "source": [
    "### Rendering Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iFZ1eIgjHwXw"
   },
   "outputs": [],
   "source": [
    "done = False\n",
    "state = env.reset()\n",
    "while not done:\n",
    "    qs_list = agent.get_qs(state)\n",
    "    action = np.argmax(qs_list())\n",
    "    next_state, _, done, _ = env.step(action)\n",
    "    state = next_state\n",
    "    env.render()\n",
    "\n",
    "env.close()\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "1_dqn_keras_mountain_car.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python (spinningup)",
   "language": "python",
   "name": "spinningup"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}