algorithms/7.A2C [discrete].ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 6048,
     "status": "ok",
     "timestamp": 1603339592882,
     "user": {
      "displayName": "Nimish S",
      "photoUrl": "",
      "userId": "16860995465865824316"
     },
     "user_tz": -330
    },
    "id": "0dfRgaa92KdI"
   },
   "outputs": [],
   "source": [
    "# %%capture\n",
    "# !pip install box2d-py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "executionInfo": {
     "elapsed": 2404,
     "status": "ok",
     "timestamp": 1603294008155,
     "user": {
      "displayName": "Nimish S",
      "photoUrl": "",
      "userId": "16860995465865824316"
     },
     "user_tz": -330
    },
    "id": "xry67bcZ2ie5",
    "outputId": "49580b45-60b4-438d-aa47-319faf4d047f"
   },
   "outputs": [],
   "source": [
    "# T.cuda.get_device_name()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "executionInfo": {
     "elapsed": 10323,
     "status": "ok",
     "timestamp": 1603339597168,
     "user": {
      "displayName": "Nimish S",
      "photoUrl": "",
      "userId": "16860995465865824316"
     },
     "user_tz": -330
    },
    "id": "ckjnmhgQ2cvM"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import cv2\n",
    "import gym\n",
    "import time\n",
    "import warnings\n",
    "\n",
    "import numpy as np\n",
    "import torch as T\n",
    "import torch.optim as optim\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "M-bHj8PV3Cir"
   },
   "source": [
    "## **Memory**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "executionInfo": {
     "elapsed": 10321,
     "status": "ok",
     "timestamp": 1603339597171,
     "user": {
      "displayName": "Nimish S",
      "photoUrl": "",
      "userId": "16860995465865824316"
     },
     "user_tz": -330
    },
    "id": "XlHumQ323CsT"
   },
   "outputs": [],
   "source": [
    "class Memory:\n",
    "    def __init__(self):\n",
    "        self.device = T.device('cuda:0' if T.cuda.is_available() else 'cpu')\n",
    "        self.states = []\n",
    "        self.actions = []\n",
    "        self.logprobs = []\n",
    "        self.entropies = []\n",
    "        self.rewards = []\n",
    "\n",
    "    def store_step(self, state, action, logprob, entropy, reward):\n",
    "        self.states.append(T.tensor(state, dtype=T.float32).to(self.device))\n",
    "        self.actions.append(T.tensor(action, dtype=T.float32).to(self.device))\n",
    "        self.logprobs.append(logprob) # HAS GRAD\n",
    "        self.entropies.append(T.tensor(entropy, dtype=T.float32).to(self.device))\n",
    "        self.rewards.append(T.tensor(reward, dtype=T.float32).to(self.device))\n",
    "    \n",
    "    def clear(self):\n",
    "        del self.states[:]\n",
    "        del self.actions[:]\n",
    "        del self.logprobs[:]\n",
    "        del self.entropies[:]\n",
    "        del self.rewards[:]\n",
    "\n",
    "    def __getitem__(self,i):\n",
    "        state = self.states[i]\n",
    "        action = self.actions[i]\n",
    "        logprob = self.logprobs[i]\n",
    "        entropy = self.entropies[i]\n",
    "        reward = self.rewards[i]\n",
    "        return(state, action, logprob, entropy, reward)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.states)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "piqynVGt3DQu"
   },
   "source": [
    "## **Network**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "executionInfo": {
     "elapsed": 10320,
     "status": "ok",
     "timestamp": 1603339597175,
     "user": {
      "displayName": "Nimish S",
      "photoUrl": "",
      "userId": "16860995465865824316"
     },
     "user_tz": -330
    },
    "id": "4sAqpONN5xB3"
   },
   "outputs": [],
   "source": [
    "class ActorNetwork(nn.Module):\n",
    "    def __init__(self, lr, observation_shape, n_actions, fc1_size, fc2_size, model_name, model_dir):\n",
    "        super().__init__()\n",
    "        self.model_dir = model_dir\n",
    "        self.model_file = os.path.join(self.model_dir, model_name)\n",
    "        self.fc1_size = fc1_size\n",
    "        self.fc2_size = fc2_size\n",
    "\n",
    "        # ACTOR\n",
    "        self.actor = nn.Sequential(\n",
    "            nn.Linear(observation_shape[0], fc1_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(fc1_size, fc2_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(fc2_size, n_actions),\n",
    "            nn.Softmax(dim=1)\n",
    "        )\n",
    "\n",
    "        # UTILS\n",
    "        self.device = T.device('cuda:0' if T.cuda.is_available() else 'cpu')\n",
    "        self.optimizer = optim.RMSprop(self.parameters(), lr=lr, alpha=0.99)\n",
    "        self.to(self.device)\n",
    "\n",
    "    def forward(self, state):\n",
    "        return self.actor(state)\n",
    "\n",
    "    def save_model(self):\n",
    "        print(\"[INFO] - Saving model\")\n",
    "        checkpoint = {\n",
    "            'model_state_dict': self.state_dict(),\n",
    "            'optimizer_state_dict' : self.optimizer.state_dict()\n",
    "        }\n",
    "        T.save(checkpoint, self.model_file)\n",
    "    \n",
    "    def load_model(self, cpu=False):\n",
    "        print(\"[INFO] - Loading model\")\n",
    "        \n",
    "        map_location = T.device('cpu') if (cpu) else None\n",
    "        \n",
    "        checkpoint = T.load(self.model_file, map_location=map_location)\n",
    "        self.load_state_dict(checkpoint['model_state_dict'])\n",
    "        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "executionInfo": {
     "elapsed": 10315,
     "status": "ok",
     "timestamp": 1603339597177,
     "user": {
      "displayName": "Nimish S",
      "photoUrl": "",
      "userId": "16860995465865824316"
     },
     "user_tz": -330
    },
    "id": "uh4XV7fC3F15"
   },
   "outputs": [],
   "source": [
    "class CriticNetwork(nn.Module):\n",
    "    def __init__(self, lr, observation_shape, n_actions, fc1_size, fc2_size, model_name, model_dir):\n",
    "        super().__init__()\n",
    "        self.model_dir = model_dir\n",
    "        self.model_file = os.path.join(self.model_dir, model_name)\n",
    "        self.fc1_size = fc1_size\n",
    "        self.fc2_size = fc2_size\n",
    "\n",
    "        # CRITIC\n",
    "        self.critic = nn.Sequential(\n",
    "            nn.Linear(observation_shape[0], fc1_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(fc1_size, fc2_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(fc2_size, 1),\n",
    "        )\n",
    "\n",
    "        # UTILS\n",
    "        self.device = T.device('cuda:0' if T.cuda.is_available() else 'cpu')\n",
    "        self.loss = nn.SmoothL1Loss()\n",
    "        self.optimizer = optim.RMSprop(self.parameters(), lr=lr, alpha=0.99)\n",
    "        self.to(self.device)\n",
    "\n",
    "    def forward(self, state):\n",
    "        return self.critic(state)\n",
    "\n",
    "    def save_model(self):\n",
    "        print(\"[INFO] - Saving model\")\n",
    "        checkpoint = {\n",
    "            'model_state_dict': self.state_dict(),\n",
    "            'optimizer_state_dict' : self.optimizer.state_dict()\n",
    "        }\n",
    "        T.save(checkpoint, self.model_file)\n",
    "    \n",
    "    def load_model(self, cpu=False):\n",
    "        print(\"[INFO] - Loading model\")\n",
    "        \n",
    "        map_location = T.device('cpu') if (cpu) else None\n",
    "        \n",
    "        checkpoint = T.load(self.model_file, map_location=map_location)\n",
    "        self.load_state_dict(checkpoint['model_state_dict'])\n",
    "        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ktQPIXcB3IxU"
   },
   "source": [
    "## **Agent**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "executionInfo": {
     "elapsed": 10314,
     "status": "ok",
     "timestamp": 1603339597179,
     "user": {
      "displayName": "Nimish S",
      "photoUrl": "",
      "userId": "16860995465865824316"
     },
     "user_tz": -330
    },
    "id": "Yq6QBFbq3I5z"
   },
   "outputs": [],
   "source": [
    "class A2CAgent:\n",
    "    def __init__(self, observation_shape, n_actions, actor_lr, critic_lr, fc1_size, fc2_size, gamma, \n",
    "                 learn_interval, algo_name, env_name, model_dir):\n",
    "        self.observation_shape = observation_shape\n",
    "        self.n_actions = n_actions\n",
    "        self.A_LR = actor_lr\n",
    "        self.C_LR = critic_lr\n",
    "        self.GAMMA = gamma\n",
    "        self.memory = Memory()\n",
    "        # MODEL PARAMS\n",
    "        self.move_counter = 0\n",
    "        self.learn_interval = learn_interval\n",
    "        self.algo_name = algo_name\n",
    "        self.env_name = env_name\n",
    "        self.model_dir = model_dir\n",
    "        # A1\n",
    "        self.actor = ActorNetwork(actor_lr, observation_shape, n_actions, fc1_size, fc2_size,\n",
    "                              model_name = env_name+'_'+algo_name+'actor',\n",
    "                              model_dir = model_dir)\n",
    "        # C1\n",
    "        self.critic = CriticNetwork(actor_lr, observation_shape, n_actions, fc1_size, fc2_size,\n",
    "                              model_name = env_name+'_'+algo_name+'critic',\n",
    "                              model_dir = model_dir)\n",
    "\n",
    "    def get_action(self, observation):\n",
    "        observation = T.tensor([observation], dtype=T.float32).to(self.actor.device)\n",
    "        action_probs = self.actor(observation)\n",
    "        action_dist = T.distributions.Categorical(action_probs)\n",
    "        action = action_dist.sample()\n",
    "        logprob = action_dist.log_prob(action)\n",
    "        entropy = action_dist.entropy()\n",
    "        return action.item(), logprob, entropy.item()\n",
    "\n",
    "    def learn(self, state_, terminal):\n",
    "\n",
    "        # ADVANTAGE\n",
    "        state_ = T.tensor([state_], dtype=T.float32).to(self.critic.device)\n",
    "        Q = 0 if (terminal) else self.critic(state_)\n",
    "        Q_values = []\n",
    "        for *_, reward in zip(*(self.memory[::-1])):\n",
    "            Q = reward + (self.GAMMA * Q)\n",
    "            Q_values.insert(0, Q)\n",
    "        \n",
    "        Q_values = T.tensor(Q_values).to(self.critic.device)\n",
    "        V_values = self.critic(T.stack(self.memory.states[:]))\n",
    "        advantages = Q_values - V_values\n",
    "\n",
    "        # CRITIC LOSS\n",
    "        critic_loss = advantages.pow(2).mean()\n",
    "\n",
    "        # ACTOR LOSS\n",
    "        logprobs = T.stack(self.memory.logprobs[:])\n",
    "        actor_loss = (-(logprobs * advantages.detach())).mean()\n",
    "\n",
    "        # ENTROPY\n",
    "        dist_entropy = T.stack(self.memory.entropies[:]).mean()\n",
    "\n",
    "        loss = actor_loss + (critic_loss * 0.25) - (dist_entropy * 0.01)\n",
    "        self.critic.optimizer.zero_grad()\n",
    "        self.actor.optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        self.critic.optimizer.step()\n",
    "        self.actor.optimizer.step()\n",
    "\n",
    "  \n",
    "    def save_models(self):\n",
    "        self.actor.save_model()\n",
    "        self.critic.save_model()\n",
    "\n",
    "    def load_models(self, cpu=False):\n",
    "        self.actor.load_model(cpu)\n",
    "        self.critic.load_model(cpu)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SrqMm67bZOXQ"
   },
   "source": [
    "## **Training**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "executionInfo": {
     "elapsed": 10312,
     "status": "ok",
     "timestamp": 1603339597180,
     "user": {
      "displayName": "Nimish S",
      "photoUrl": "",
      "userId": "16860995465865824316"
     },
     "user_tz": -330
    },
    "id": "JBRyqiKgZNry"
   },
   "outputs": [],
   "source": [
    "env_name = \"LunarLander-v2\"\n",
    "env = gym.make(env_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "executionInfo": {
     "elapsed": 19678,
     "status": "ok",
     "timestamp": 1603339606549,
     "user": {
      "displayName": "Nimish S",
      "photoUrl": "",
      "userId": "16860995465865824316"
     },
     "user_tz": -330
    },
    "id": "5pRCqoCL4_rx"
   },
   "outputs": [],
   "source": [
    "agent = A2CAgent(observation_shape=env.observation_space.shape,\n",
    "                 n_actions=env.action_space.n,\n",
    "                 actor_lr=7e-4,\n",
    "                 critic_lr=7e-4,\n",
    "                 fc1_size=400,\n",
    "                 fc2_size=300,\n",
    "                 gamma=0.99,\n",
    "                 learn_interval=5,\n",
    "                 algo_name='A2C',\n",
    "                 env_name=env_name,\n",
    "                 model_dir='./')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "executionInfo": {
     "elapsed": 19679,
     "status": "ok",
     "timestamp": 1603339606554,
     "user": {
      "displayName": "Nimish S",
      "photoUrl": "",
      "userId": "16860995465865824316"
     },
     "user_tz": -330
    },
    "id": "QNKNh8pZaQx9"
   },
   "outputs": [],
   "source": [
    "N_EPISODES = 3000\n",
    "\n",
    "episode_rewards, episode_lengths, mean_rewards = [],[],[]\n",
    "best_reward = -np.inf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NWn6_gG0aVhj"
   },
   "outputs": [],
   "source": [
    "for episode_n in tqdm(range(N_EPISODES)):\n",
    "    total_reward, total_moves = 0,0\n",
    "\n",
    "    done = False\n",
    "    observation = env.reset()\n",
    "\n",
    "    while not done:\n",
    "        agent.move_counter += 1\n",
    "\n",
    "        action, logprob, entropy = agent.get_action(observation)\n",
    "        observation_, reward, done, _ = env.step(action)\n",
    "\n",
    "        agent.memory.store_step(observation, action, logprob, entropy, reward)\n",
    "        total_reward += reward\n",
    "        total_moves += 1\n",
    "\n",
    "        if ((agent.move_counter % agent.learn_interval) == 0):\n",
    "            agent.learn(observation_, done)\n",
    "            agent.memory.clear()\n",
    "        \n",
    "        observation = observation_\n",
    "\n",
    "    # IF DONE BUT MEM HAS ELEMENTS\n",
    "    if (len(agent.memory) >= 2):\n",
    "        agent.learn(observation_, done)\n",
    "    agent.memory.clear()\n",
    "        \n",
    "    episode_rewards.append(total_reward)\n",
    "    episode_lengths.append(total_moves)\n",
    "\n",
    "    mean_reward = np.mean(episode_rewards[-100:])\n",
    "    mean_rewards.append(mean_reward)\n",
    "    if(mean_reward > best_reward):\n",
    "        best_reward = mean_reward\n",
    "        agent.save_models()\n",
    "        \n",
    "    print(\"ITER: \",episode_n,\"\\tRWD: \",round(total_reward,2),\"\\tM_RWD: \",round(mean_reward,2),\"\\tLEN: \",total_moves)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "meGYOarJpEk9"
   },
   "source": [
    "### **Results**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 305
    },
    "executionInfo": {
     "elapsed": 1838,
     "status": "ok",
     "timestamp": 1603341639269,
     "user": {
      "displayName": "Nimish S",
      "photoUrl": "",
      "userId": "16860995465865824316"
     },
     "user_tz": -330
    },
    "id": "Z3wLdlDNl47E",
    "outputId": "04ab7fbb-854a-4bd9-bde2-c7c7570749c1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "186.15663016244588"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "708625"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU1dnA8d+TfU+AhABZSNhFUIQIuMsiImrRtlrta12qpVrUVttaLLbaoq1tfbtYrb4uVK3WpVYrCopgXXBBBAVkEQh7AoQkkH1PzvvHvTOZJDPJJDPZZp7v55NP7px75865mckz555VjDEopZQKLiG9nQGllFI9T4O/UkoFIQ3+SikVhDT4K6VUENLgr5RSQUiDv1JKBSGfg7+IZIjIuyKyTUS2isgP7fSBIrJKRHbZvwfY6SIiD4pIrohsFpHJvuZBKaVU5/ij5N8A/NgYMx6YDiwUkfHAIuAdY8xo4B37McAFwGj7ZwHwiB/yoJRSqhN8Dv7GmMPGmM/t7XJgO5AGzAeetg97GrjE3p4PPGMsa4EkERnqaz6UUkp5L8yfJxORLOAU4FMg1Rhz2N51BEi1t9OAgy5Py7PTDrukISILsO4MiI2NnTJu3Dh/ZlUppQLehg0biowxKe72+S34i0gc8G/gR8aYMhFx7jPGGBHp1DwSxpjHgMcAcnJyzPr16/2VVaWUCgoist/TPr/09hGRcKzA/5wx5hU7ucBRnWP/Pmqn5wMZLk9Pt9OUUkr1EH/09hHgSWC7MeaPLruWAdfY29cAr7mkX233+pkOlLpUDymlApAxhqYmnUSyL/FHyf8M4DvATBHZaP/MA+4HzhORXcBs+zHACmAPkAs8DvzAD3lQSvUR2w+X0egS6JuaDGfc/19G/HxFL+ZKteZznb8x5kNAPOye5eZ4Ayz09XWVUn3PtkNlzHtwDQA3nTuSy3MymPHAe8799Y1NhIfq2NK+QN8FpZTf7C+udG4/8t7uFoEf4OPdxT2cI+WJBn+llN8cLq1pd/9L6w+2u1/1HA3+Sim/cQT3P3zzJGfag1eewtu3nQ3AF/uP90q+VFt+HeSllApuVXWNAFyWk8GguAie+WQ/545NISEqnFtmjuJv7+2mpr6RqPDQXs6p0pK/CmqN2v3Qr8pq6vn2tEwAZo5L5anrppIQFQ7A2CHxNDYZlryxrTezqGwa/FXQ2pxXwsifr+CNzYd6OysBobK2gZKqetIHRLvdf84Ya5aB5z490JPZUh5o8FdBa+0eq+fJYx/s6eWcBIadBeUApCW5D/7xUeGcOzaFiNAQrB7fqjdp8FdB62hZLQAVtQ29nJPes6+okifW7OHDXUU+n2vb4TIAhg+K9XjM2aNTqGts4lhlnc+vp3yjwV912SPv7WbiPSspr6nv7ax0ycptRwDYU1jJLc9/QWlV/7yOrlq26RDnPvAe9y7fzlVPfsqOI+U+ne9IaQ0icOKwBI/HjBwcB8DuwkqPx6ieocFfdUljk+F3b31FeU0DL6zrf323a+obOXis2vn49U2HuPRvH1EZRHcBf169s8XjfcWV1DU0dfl8ecerGZoQ1e4I3pEp1l3BrqO+fdEo32nwV11yqKQ5cJb3w4D5TzeNjnuKKpn+23d6ITe9Y0hCFAA/mTMGgO//YwOn3/8ODY1d+wLIP15NmofGXodhidEkxYSz6WBJl15D+Y8Gf9UpxhgefjeXlVuPONMOFPe/W/gQezaqf990OhFhzf8G5TUNPpV++4tPdhezp7CSGWNTWDhjFPGR1pCfooq6DkfpepJfUu2xsdchJEQYlRLHgWNVXXoN5T8a/FWnHC2v5Q8rd3Dv8u0AZCfHsr8f/iPvP1ZFTEQokzOT+Gzx7Bb78kuqaWoyXPzXD8latDzgqoIKy2u58vG1HCmrIWNgDCLC8wumE20PvNrXhS/zmvpGK/h3UPIHyBwY0+11/u/vLOTHL23qM9NIG2P4wXMbeH5d3+nmqsFfdcr7OwpbPJ6WPZD9xf0v+O8tqiRrUCwiQmJ0OB8vmsnieScAMOOB97j4oQ/5Mr8UgBPvXtmbWfU71/r2jAExAExIS+S/PzkHgAfe3un2eZ5U1zXyp1XWc9rr6eOQlRxLYXlti6pDfyqrqeeapev49+d5/OWdXd3yGp1VWF7Lii+PcOcrX/Lx7qI+MbhQg7/qUG1Do7Nf9r82NDfunpKZRFZyLMcq6yjrRz1+jpbV8N6OQrKSY5xpw5KimTGueanTrYfKeiNrPeKBlTuc25OHD3BuD02MJjs5lk0HS9hTWOH1+SYvWcX/2WMlLpgwpMPjYyKsO4xbn//C69fojM9d5g/qK8Hf0Q0W4NuPf8rIn6/guU89rrDYIzT4q3YVltcy9q63WPTvL6mobWDD/uPMPmEwyXER3HvJBLIGWQH0QD8q/TsCwuTMAS3SR6bEuT0+LESo72IjaF9jjOHzA1Zj638WnsGU4S3/Bj+aPRqApR/t9ep85TX1VNdb8/nccGY28fZUDu25+ORhANR3U+l3j0uVUnZyx3ciPeGBt3e0SVv86haOdLF9xR80+Kt2Ofp+v7j+IHe8vIkmA985LYv1d53HicMSnbf5Xakn7i2O6QW+c9rwFukiwpo7ZnDb7DHOtPu/PpGGJuN2EFR1XSO3v7SRg/2kzeNAcRXZd1qraX1jcjqTMpLaHDN/UhoAz671rm56vV3K/ssVk7jrovFePSfV7mXUXT1+DhyrIi4yjBvOzOZQSXWXey/5U0WN1W70NfuLz8FRtdgbNPirdhVX1jq3V3xp9fCZmjXQmTbcLvl3tt7//Z2FPLGm56dVKK1urp6KDGs7s2TGwBhunTXK+XjOiVY1xurtBW2OfWLNHl75PJ/73/qqRboxxqfpC4oranl/ZyHGGL7+t4+4b7nvE6EVVdRy9h/edT6+4axsj8d+fbL1BVBQ1nGp9NXP84mLDOPcsYM7lZ+zRicD1heoP+0prODZtfsZPiiGiemJ1DY0saOgd8cUNDQ2cai0hu+dlc0fLjuJDXfN5sOfzQDge8+s9/vfwFsa/Pupnpgb5Yk1e/jhCxvbpEdHNAfNmIgwBsdHsq/I+5J/fWMT1yxdx73Lt7cIxj1h/kMfAlaJ3hMRYduvz2fd4lkMjI3gpPRE5zxAxyrr+MRejeqhd3MBWL75MDV21UdVXQMn/PItfvV61wP2rD++zzVL1zHxnrf5/EAJj6/Z6/P7ffr9/23xeGxqvMdjL5uSAcBXHYz4La2uZ9mmQ5yaNYDE6I6re1q8Ro71GgeP+++uqbiilpn/+z4NTYZRg+OcdzYf5fo+dYUv9hRZg+fGD0sgMiyUQXGRpCVFc8JQayT0/mO9c9eswb8f2nqolLF3vdXt3cYc3TkBtvzqfM4bn8o9F7e9tU+Jj+SNzYe9Pu+/N+Q5t9/a4v3zPPE2MBpj2GffoXxzSnq7x1pfalb1RGJ0OLsLK2lobOKqJz7lysfXUlPf2CLgfWHXo28/XEZNfRNPfbyPj3d3PugcLa+hxJ5mwnXOoQ98mHunsrZ57MJbPzqL9XfNJiTE07LbMG6I9cWw40gZpVVWz5mth9pWT3zvmfVA16ZqyLC7hC55Yxuf7C72S9XM2b9vvrO57oxsMgdad6W/f6ttfXtHjlXW+a1Hzja788D4oYnONBFxFkBcR5r3JA3+/YwxhqufXEddYxN3vvJlt76Ow/JbzyQuMozHr87h2jPaVhdsPVRGdX0jf3svt8NzPrFmD4tc8p13vGsf/KYmw1tbDpO1aDnZd65ot9tgaXU92XcuZ9U2q+pm8bwTCOvEIuKOqYgfeHuns9fGXf/ZwtHyWk7JtEqXR8utKhLHlwDAf77I79xFAZ/tdb/S1d5O9L5p7Xd2tVTmwBjGDUkgOS6y3eMHxEYQGRbCPz89wLJN+by/s5D7XAoCDo4vlCWXTOh0nhyBec2uIq58fC1XL13X6XO0VmlXn3xy50wmZSQhIsREhNLQZJx3Zt6oqmvg1PtWc+OzG3zOE1g9fSLCQhiR0rLx2fE36K0Bbxr8+5FDJdVk37mCYpcZEesamthXVOmsluiqD3cV8dB/m7vFHS236vp/9bUTOXFYoqenAfDYd6YAVgnr8kc/8XjcuzuOOu8mrpyaydjUeJ7+eB+n3reaB9/ZxT8+2ceDbrrmNTYZvvvUZ2QtWk7WouW8s72AET9fwY3Pfu485q0tR9o8z+GJNXswBhb8w/pndrRTeOsUu1fQo+/vdqa9bN+9fO+sEUDzl9ibLvl4aX3zHY63Nh48TkRYCH/7n8mA1fsmMiyED7tYdWFMc2P1uz851+vn1Tc2sa+4iqUf7QNwGzwjwkLIGT7A+eXYGQNjI1o89nVhd0dvs7suPIGhic0DzRacbb0/3/+Hd4F8c14Jj763m8Ymw6ptBV61e3Rk26EyxqbGt5nzKCkmnLjIsHa71XbneAC/BH8RWSoiR0Vki0vaQBFZJSK77N8D7HQRkQdFJFdENovIZH/kIRi4C3DjfvEm5z7wHlc8tpaiilo3z+pYY5Phqic/5YG3d1Jsn8MxN/voVPfdH13NOXEIsXY7wLp9x9icV+I2WHywszmARYaFMPOEwZTVNFBYXssfV+3kF69t5Y+rdrb5wO8urOC/Xx11Pr7+6fVtzv1/H+xuk+ZQ39jyfN4MRHLVujukw7WnZ3HBhCEkx0WQZ9ddf3HgeIsS3ppdhW6f6055TT2Pr9lL5sAY5k0cSu59F3DLzNEMSYzqcqNl9p0r2FNUycS0RELbqeppzdHjaa/dluMoDLjaW1TZ5a6UIsJ9l7a8Y/ClXcMx3chJ6S17MJ1qd054f2fH78MdL2/iaw99xIP/bb6DXbf3WJfzBNY1bTtcxvihbWc6FRFGDY5rt8fPbS9u5PqnPvMpD574q+T/FDC3Vdoi4B1jzGjgHfsxwAXAaPtnAfCIn/IQ8Fx73jhutV3jZFcHJn26t7nUNeXe1eQeLWdXgVUaGT3Yc8Ogqzd/eLZz+2sPfcRtL7ZtKD5cWk10eCjnjEnhhrOy+dnccW7P9UGrf9R8D1U6b9xyJj89fyxhIUKIeA5sriWrEIExXnyhtTY1u7mH02/tutqbzh2JiJA2IIb9xVVU1DbQZGD04DhW3279PZ7+eB/f/8f6Fl+Gnuq3r/u79U9eaAfasNAQQkOECycO5VBJTafqxf/+0V6yFi13Pj53bOdK51dMzWzxOL+kukWvlH1FlRSW15Kd0vV+9N+emslfrpjE6SMHAbS4o+2MhsYmXlp/kKjwkBbvE8AZo5Kd23ntNC7nl1S3uVMLD5UWg7O6oqCslmOVdYz3MM31xLREco9WuB1HYoxh9fYCEmM615juLb8Ef2PMB0Drr8j5wNP29tPAJS7pzxjLWiBJRIb6Ix+B7uF3rdJtRFgIl7lpsHzHTXdEb/zdvrV3uPz/1vLsWmv0YXJchJtntJU5KIZ5E5tHd77Z6i6lqcmwcmsBkzKSePq7U0m3pxVYMv/ENv3NWy/zl29Xqay9cxZ7fzuPv197Km/fdjYT0hJZOGMUV5+WRUlVfYs7BmMML6w7QH5JNbtdgn+TsUpcnfXs9dP4bPFs9v52HldOzWTf/Rc6+6unJ0Xz8e5i1thfWt+cksEo+0tz9fajrNxa4BxYtnZPMaMWv8nmvLZ93B13WW/96KwW6RkDY2hsMp2acM21t9G1p2fxI5exC95IiY/khQXTefu2s3no26dgDOwpav47fmJXM07ooEqwPSLC/ElpfP+ckQBu/ybeGHPXm+w6WkFNvfsvx0evsqolt7RTwt5sjzkYkRLLP2+Yxqa75zAiOY6dPq5xsO2w9Zqegn/mwBiq6hqZ++cPWtz5HDxWxZI3tlNV1+h2PIY/dGedf6oxxtGV4wiQam+nAa4TwOfZaS2IyAIRWS8i6wsLvb91DlSOlY+GJUaxY8lcosJDeen7p7U45plP9nfp1vnz/cf5Vk4Ga++cRWxEKMcq69hj3+53JlD+7X+mtHjsOqlWgd0gOjQxqsUx3zkti1d/cDpnj0nh9984ia+dPIz3dx5tEcgPlVQTFiKkxEciIswYN5gxLl0VJ6YnUF3f2OKfe0dBOYte+ZKbnt3AgWNVXDW9ZUm2syLCQpyv35qjWuim56w2iAlp1j/6yS7/tEl2z6A3v7T+JT5xU8ddVt3AiOTYFnXW0Dz/Tme6RYaHNufzFxeN71SVj8P0EYMYkxrvvPt7yKU6xNHZwFFq98VJadYXyDvbj3ZwZFs19Y3Ou9+FM0a6Peb0UVYe22tYdRRWXr/5TE4flUxidDhjhsR32N21I46ePo4eVK05qs12F1aSfecKXlh3gPKaes76/bvOUdaOhmF/65EGX2NFpE5FJWPMY8aYHGNMTkpK5xuUAo2j9HrvpROcAWhq9kAevWoKf/7WJOftbmen433l8zyKK+sYnhzDkMQoXnT5QrnxHPf/TO2ZOa55sI9rsHI0yF1ySpvveUSEZ747lctPzeC0kYOobzQcKqnmyQ/3UlBWQ35JNUMSozwGsDNHpRAi8MvXtjirRhwjkzfnlVLfaDgpPYm7Lx7P0mtzOn1NHZk+omUAdATv1xaewXt2I6vjfXE0FLsLKhv2H3c7K6bjn99TTyBXuwrKuezRj6lvNNw6azT77r+wS4Hf1Sh79a03txzhSGkNN7n0gulMrylPBsRGMC17IOv3dXx9rY37xVuA1Yb0kzlj3R6TEBVOTERou4sOvbH5EKkJkcTaU1uDFbDzS6o71W7T2qPv7yExOtzjtBezx6dy+3nNd2WLXvmSp1rdiffH4F/gqM6xfzu+1vOBDJfj0u001Q5Hj41RKS1LEHMnDOGSU9KcH6Dco53rEuhYvNwRwFxLKK1L6d548MpTnD1Vlm085Ex3lLo6+iAPt/cv//IwS97YxrTfvMNrGw8xON5z98SU+EiuPi2LTXmlzoFXmw62vMXPGhTLdWdkM3NcqrtT+MR1grgRrRpAs5JjGZMaR35JNcYYZ3fEV7/Id94ZVdQ2cPFfP+RIWY3bkdLpA6IZkhDFn1bvdJYkPbn5n1/wmR1ET/FTdYHrl8f0377jLCX/5YpJfjk/QFiosKOgvMUYkI643uWuuWNGu3epUeGh7CmqdFs1uqugnCYDs09IbfMcgO882bVuqLe/uJGK2gYSosPaPe7WWaNbPH7q430A/PT8sVwwYUinOyh4qzuD/zLgGnv7GuA1l/Sr7V4/04FSl+oh5UZtQ6OzztjTfOkZdtD01DjqSYgI07IHOic5CwsN4Y1bzuSCCUM6HAjlTlxkmPOfaG9RJVvySymrqefgsSpCxJo9sz3D7eDpulgM4JyMzJOrplvz9Px5tfV3al1FcmqW+x47/hATEebs4fPazWe02R8bGcaqbQU880nLWRwddeg3/mODs8dHfFTbQBESIlx+qlVemvfgGrd5eOS93by+6VCL8Q5T/HjNN53b8i5w5Y/Ods4D5A+Oz8wbmw91cKSluKKWR+yut3ddeAKDE9ovqDx7/TQA51gPgJIqqyrVMUjtopNazrtzeU7z57+z1anGGF6xx3kssLsDt2fPb+bx3A1WHh0N3wtnjOKRq6b4fOfmib+6ej4PfAKMFZE8EbkeuB84T0R2AbPtxwArgD1ALvA48AN/5CGQHS1r7uXj6YOQGh+JiFUX+8vXtrg9pjVjDAePVTG2VX3khLREHrlqSotb4M6ICAvh1KwB7C2u5KK/fshJ97zNgWNVDE2MbrFqljvD7LuNL1oF+44GEmUMbP5Sqa5rJP94NdOyBzIxLZHXbz6zS428nbH6tnPY9uvz3d7eO67l7mVbgeYS86aDpWQtWt6iD78jSLW2cMZI5+pjVy9d12KBma+OlPG7t77ilue/oLy2gZvOHckLC6aT4MUMm95KajV9Q+vPjK+uPT2LhKgwar1cRW3KvaudI3e9qRYZPyyBSRlJzjvQLw4cZ9KvV/H8ugPOwVyt6+Xjo8Kdn7vsO1d0qmC1/XBztd4FEzvuzxISIi16JvUEf/X2udIYM9QYE26MSTfGPGmMKTbGzDLGjDbGzDbGHLOPNcaYhcaYkcaYicaYtp22VQuv26Whm2eM8nhMWGgIjsLJM5/s92oFo9LqesprG7qlTnHskPiWo103HiLdi1WeHIurOFx3RhZnjBrEt6e232AbGRbKI3Z1066j5eQdt77UXr/lTCamd71HirdCQoSYCPdflq4BPSxEmGvPef/jf21yps+fNIx991/IgFj3vasiw0J5/6fWZGAf7Czkj/biKb9dsZ25f255N/DDWaPbtEP46sKThnLOmBRW3HoWHy+a6ddzQ3Of9493Fzvne6praPJqKm1vxxoMHxTDx7uLqaprYIXd8O5azZTkpktlhstn9sk13k1zDbDJ7rm0+vazOxxR7er6M60R9NNadVntDl0r2qke5SjhXHdGltfPyS+pdlYFeeJofOxqCb89J6cn8Swtu2zOOsG7mR//s/AMrnriU/5x/VRGeJhj3x1HafSW57+grKahw/Vke8qZo5P5/TdP4o6XN/P2bWcTGRbKhLQEtuRb9fcT0xL5zaWeJ5pzyBgYw/1fn8iiV77kne0FXHt6lnMRFYefzR3nrKv2p/QBMTz93al+P6+r+ZPS+PxACU+s2cPLG/KcjeR/vPxkpgwfQGpCFFHhoS2qYB69aoqzQbojji+Jafe9Q7l95+SYkvrKqZlu7w4npjUXHCpqvZ+EcHNeCfFRYR7XiPDkFxeN56rpw9tt4/IXnd6hHxiRHMt541MZ1EEJwrVedqMXc6U7FpK48CT/D7MY7WbWyAVne9d7KDs5lo8WzexU4IfmkbuORlPHWIK+4PKcDPbdf6Hzmv5+bXMgff2WM73+Ar5iaiZTsweyr7iKs1wmMjtvfCo77p3L98/uuH65r5o93qr3/+t/c1v0Wrv9pU2c84f3nD17HG0n38rJYO6EIV5X6TnasMrdrMl8r4dqxUFxkey7/0Jyhg/gpfV5ZC1azuHS9qt/mpoMb2454pxfqLOyk2O7pUDWmgb/Ps4Yw6HSamcvmPb8bO44vlpiDbR23Na2Z3dhBeOGxPu1btjhZLuqZXJmEvvuv5B991/o99doLTREWiyWMSyp872VekpKfCT/vGEay289s9PP/cbklg2tX94zh8evziEyLLTd2Tr7urSkaCI7aBNa8eVhNudZjeO/cDPDbHvSB8S06KF05wXWCPN7L5nQYaOqa2HKtdG4tcYmw60vfEFJVT0zOrnGQU/Tap8+rriyjpr6Jq/qy6G5e9qbW45wtLzGOS1xa+U19by7o5BL3fS79wcRYc9v5tHN7axtPHjlKfxgxkieW3uACWndX9fvi9O72MD3rVMzyckayKz/fR/Aq6UT+wtHg+8Dl53MN6ek85fVu/jT6p1cekoar36Rzw/sgXQT0xKJ60LpeP6kNCZnDmBYUjShIcKV0zK9KvxkDoxxDnz85Wtbufq0rDbHGGNYv++Yc3rz88b7v1uxP2nw7+Mcs0WmdaIK46fnj+UPK3eQW1DhNvjX1Dcy8Z63Af/32nDVW6XQcUMSujTNcH8yMiWOW2eNZmA3zfvSW1ITIikoq2WyPVX2rbNGceO5I4gIDeFVlymyj/gw26ZrW5i3d70vfv803t9ZyE/sRvqquoYWDfxz/vQ+Owuax9j8bO64DtvceptW+/RxjnltOtN4OX+SVfWx28PqWq596F1H5Kr+5fbzxrhdX6E/u+vC8Vw2JZ0su/1GRIgMC0VEeO6GaZxp3y39+msn9mi+UuIj+eaUdJ642hoh/qtl25zdbRsam1oEfoAbz+n7bS9a8u/j8kusxktPg7vccUwv8Iv/bOGyKekten/UNjQ6l2bc/Zt53TaARKmuuPjkYVzcapFzhzNGJfd4X/jWHBO0vbj+IG9sPsTWX891Vgc53DpzVLePK/EHDf59XN7xauKjwjq1RqprQN9bVOlcK3TjwRIuefgjAP5nWqYGfqU6yXXKk8q6Rl79Is+57OZP5owh92gFt3uYY6iv0eDfh1XXNfLejsIudVl884dnccFf1rCzoJwrHlvL4nkn8KDLSl2eJsFSSnkmIvx83jie/ng/+SXV3Pai1QaQHBfJzTNHd/DsvkXr/Puw655ax4FjVYztwuIj2cmxhAi8sO4gpdX13PHvzc7G42evn+ZxJKlSqn0Lzh7JR4tmOtdvBlp0Me4vNPj3URf9dQ1r91jr4yy+sHP9mcHq8jksKdq56IZDiFgjTpVSvnn+e9Od2z89v//dSWu1Tx9kjHEO/T93bAopXRzqPSAmwlnaBzhtxCDnzIFKKd9EhYfyrxtPIyUukugI/0+p0d00+PdBrsPPHVMtd8U3Jqc5pwr+yl79SynlP44F4vsjrfbpgw6XWANYrpyawS0zPc/k2ZFT7ZkBo8NDNfArpVrQkn8fVGCPXvzG5HSf+gufOCyRbb8+3+NUw0qp4KUl/z7IEfxTO1idyBsa+JVS7mjw74N++vJmgC439CqlVEc0+PcxO440L/+m9fRKqe6iwb+P2WvPE3Lt6Vm9mxGlVEDTCuE+oqSqjsOlNSx+9UsAFrazXq9SSvlKg38XXf/UZ6QPiObui09kY16JT/3xAb71f2vZUdBc5TNIp19QSnUjDf5dUFPfyDtfHQUgc1AsS97YxvfOyu7SNAwOroH/qumZ/Xo5PqVU39drdf4iMldEdohIrogs6q18dMWza/c7t5e8sQ2Ax9fs9cu54yPD+NXXAnsVKqVU7+uV4C8iocDDwAXAeOBKEel6sbkHldfUc+/y7W73ldXUd+mcNfWNACRGh7PhF+fpPPtKqW7XWyX/qUCuMWaPMaYOeAGY30t56ZRth8qc22fZs2MOH2TNt7/k9W1dOufRsloA7rrwBCLCtAOWUqr79VakSQMOujzOs9OcRGSBiKwXkfWFhYU9mrn2HC61Rt8uvTaHp6+byj+/N41XbjodgH9tyOviOa2ZNx3LLyqlVHfrs8VMY8xjxpgcY0xOSkpKb2fH6ZAdqKdlDyIkRDh9ZDKD4iKdE7CVd7LqxxjDgWPWOr1DEn2fzkEppbzRW7198oEMl8fpdlqfd6S0hoSoMGIjW/7pHKv6bD9cztRs71BmB+MAABTpSURBVKZ5La2u5+Rfve18nJ0c67+MKqVUO3qr5P8ZMFpEskUkArgCWNZLeemU/cVVpLlZU3dimhX873h5k9fnuuiva5zbl+eka0OvUqrH9ErJ3xjTICI3AyuBUGCpMWZrb+Sls/YWVTIpI6lNumMStn3FVRRX1DIorv1J2Y6U1nDwmFWFdM/F47n2jGz/Z1YppTzotTp/Y8wKY8wYY8xIY8x9vZWPzmhqsurn0wa4b5i95rThAOwurOzwXC9vsNq7n7ruVK7ReXyUUj2szzb49kUvfGYF7GFJ7oP/LbNGA7A5r6Td8zQ2GR54eyeD4yM5Z0yKTwu2KKVUV2jw74RV244A1tq47iTHRRIfFebsvePJTnsqhwtPGqqBXynVKzT4d8Keokrmnjik3dWxMgfGsL+4/eDv6Nd/0UnD/Jo/pZTylgZ/LxVX1LK/uMrZpdOTkSlx5B6taPcYx0CxYUnar18p1Ts0+Htp2aZDAJzspqePq7FD4skvqW53sNfiV7cAkNJBjyCllOouGvy9tP2wNafP1Kz2B3DlDLfm9b/275+5/QJ4f2fzVBVhofrnV0r1Do0+XvpkTzFzxqd2OM/+qfaXw4b9x5l4z9tt9r9rrwPwp2+d7P9MKqWUlzT4e6GhsYlDJTWMTo3r8NiQEOH288Y4HxdV1Dq3P9ldzFtbjjBqcByXnpLeLXlVSilvaPD3wuHSGhqbDBlupnVw59ZZo5k1bjAAD7+bC8CxyjqufHwtR8pqGDckvtvyqpRS3tDg74WDx62umxkDvQv+AA9eeQoAecetbp2Tl6xy7psxdrAfc6eUUp2na/h6Ic+eg8fbkj9AbGQYwwfFsGpbAfMf/siZ/sFPZ5A5yPvzKKVUd9CSvxcOHq8iRGBoJ/vlnzvGWodg08Hm6R408Cul+gIN/l44cKyKoYnRhHeya+aCc0a2ePzLi/rFMsVKqSCg1T5eeG3joS49L81lArgnr8lh1gmp/sqSUkr5RIN/BxwDtbraQ+e5G6ZRXdeogV8p1ado8O+AY5K2H9rTNXfWGaOS/ZkdpZTyC63z78DeImthlixdX1cpFUA0+HdgnyP4D9Lgr5QKHBr8O7D/WBWpCZFER4T2dlaUUspvNPh3oKiiltQEnXdfKRVYNPh34HhlHQNiIno7G0op5Vca/DuQX1JDaoIuuqKUCiw+BX8RuUxEtopIk4jktNp3p4jkisgOETnfJX2unZYrIot8ef3uVl3XSFFFLcO1sVcpFWB8LflvAb4OfOCaKCLjgSuAE4G5wN9EJFREQoGHgQuA8cCV9rF90pEya63doYla56+UCiw+DfIyxmwHEGmzutV84AVjTC2wV0Rygan2vlxjzB77eS/Yx27zJR/d5XCpNZvnEG3wVUoFmO6q808DDro8zrPTPKX3SUfLrFW4UrXkr5QKMB2W/EVkNTDEza7FxpjX/J8l5+suABYAZGZmdtfLtOt4VR0AA7W3j1IqwHQY/I0xs7tw3nwgw+Vxup1GO+mtX/cx4DGAnJwc04U8+Ox4VT0ikBAd3hsvr5RS3aa7qn2WAVeISKSIZAOjgXXAZ8BoEckWkQisRuFl3ZQHn+0rqmRwfCShIW3aNJRSql/zqcFXRC4F/gqkAMtFZKMx5nxjzFYReQmrIbcBWGiMabSfczOwEggFlhpjtvp0Bd3oeFUdQxOjOz5QKaX6GV97+7wKvOph333AfW7SVwArfHndnlJW00BClM56rZQKPDrCtx3lNfUkRGl9v1Iq8Gjwb0dxRR2D4rSnj1Iq8Gjw96C2oZHS6npS4nReH6VU4NHg70FRhdXHf7BO6qaUCkAa/D0oLLdG96bEa/BXSgUeDf4eOIN/nE7toJQKPBr8PXCs3Zs2QPv5K6UCjwZ/D46U1RATEcrAWO3to5QKPBr8PSiqqCVZe/oopQKUBn8PSqrqGRCjA7yUUoFJg78HZTX1OpunUipgafD3oKxap3ZQSgUuDf4eHK+qJ1GrfZRSAUqDvxsNjU0cr6rTBl+lVMDS4O/G8ap6jIFkndRNKRWgNPi7UVxpje4dFKslf6VUYNLg70axPambTueslApUGvzdKKuuByBRu3oqpQKUBn83ymsbAIiL1CUclVKBSYO/GxU1VvCP1/V7lVIBSoO/GxV2yT9WS/5KqQClwd+NytoGosJDCA/VP49SKjD5FN1E5A8i8pWIbBaRV0UkyWXfnSKSKyI7ROR8l/S5dlquiCzy5fW7S3ltA3GR2tirlApcvhZtVwETjDEnATuBOwFEZDxwBXAiMBf4m4iEikgo8DBwATAeuNI+tk+pqGnQ+n6lVEDzKfgbY942xjTYD9cC6fb2fOAFY0ytMWYvkAtMtX9yjTF7jDF1wAv2sX1KRW2D9vRRSgU0f1Zqfxd4095OAw667Muz0zyl9ykVNRr8lVKBrcMIJyKrgSFudi02xrxmH7MYaACe81fGRGQBsAAgMzPTX6f1SnltA2lJunavUipwdRj8jTGz29svItcCFwGzjDHGTs4HMlwOS7fTaCe99es+BjwGkJOTY9wd010qauuJj4rvyZdUSqke5Wtvn7nAHcDXjDFVLruWAVeISKSIZAOjgXXAZ8BoEckWkQisRuFlvuShO2i1j1Iq0Pka4R4CIoFVIgKw1hhzozFmq4i8BGzDqg5aaIxpBBCRm4GVQCiw1Biz1cc8+JUxxmrw1d4+SqkA5lOEM8aMamfffcB9btJXACt8ed3uVNvQRH2j0ZK/Uiqg6RDWVhxTO2g/f6VUINPg34pjUjct+SulApkG/1YqdDpnpVQQ0ODfSrmj5K/VPkqpAKbBvxVnnb9O7KaUCmAa/FupqLWWcNSSv1IqkGnwb6Wkygr+CRr8lVIBTIN/K3nHq4kIC2FgbERvZ0UppbqNBv9WdhaUkz0oFnvEslJKBSQN/q2UVtczJDGqt7OhlFLdSoN/K6XV9SRGa08fpVRg0+DfSpkGf6VUENDg78IYQ1lNgwZ/pVTA0+DvoqK2gcYmo8FfKRXwNPi7KK22+/hHax9/pVRg0+DvwhH8teSvlAp0GvxdNJf8NfgrpQKbBn8XZVryV0oFCQ3+LrTaRykVLDT4u9Dgr5QKFhr8XZRVNxAaIrqKl1Iq4Gnwd1FaXU9CVJhO6qaUCnga/F2UVtdrTx+lVFDwKfiLyBIR2SwiG0XkbREZZqeLiDwoIrn2/skuz7lGRHbZP9f4egH+pJO6KaWCha8l/z8YY04yxkwC3gB+aadfAIy2fxYAjwCIyEDgbmAaMBW4W0QG+JgHv9Hgr5QKFj4Ff2NMmcvDWMDY2/OBZ4xlLZAkIkOB84FVxphjxpjjwCpgri958Ke841UMSdC5/JVSgc/nbi0ich9wNVAKzLCT04CDLofl2Wme0t2ddwHWXQOZmZm+ZtMrpdX1DIqL7JHXUkqp3tRhyV9EVovIFjc/8wGMMYuNMRnAc8DN/sqYMeYxY0yOMSYnJSXFX6f1qLahkfpGQ1xkaLe/llJK9bYOS/7GmNlenus5YAVWnX4+kOGyL91OywfObZX+npfn71aVtY0AxGoff6VUEPC1t89ol4fzga/s7WXA1Xavn+lAqTHmMLASmCMiA+yG3jl2Wq+rrG0ANPgrpYKDr5HufhEZCzQB+4Eb7fQVwDwgF6gCrgMwxhwTkSXAZ/ZxvzbGHPMxD35RYQd/Hd2rlAoGPkU6Y8w3PKQbYKGHfUuBpb68bnfQkr9SKpjoCF9bc8lfG3yVUoFPg79NG3yVUsFEg7+tss6u9onQ4K+UCnwa/G3PrzsA6BKOSqngoMHfpgu5KKWCidZx2E4clogxHR+nlFKBQEv+tqYmQ4iu4aKUChIa/G2NTYZQjf5KqSChwd/WaAyhIfrnUEoFB412Nqvk39u5UEqpnqHhztbYZAjVhduVUkFCg7+tyRhCtM5fKRUkNPjbtOSvlAomGvxtjU1a8ldKBQ8N/rYmoyV/pVTw0OBva9B+/kqpIKLB39akwV8pFUQ0+NusQV4a/JVSwUGDv62xCUK0zl8pFSQ0+NuadISvUiqIaLiz1TU2Ea7RXykVJDTa2QrLa0mJj+ztbCilVI/wS/AXkR+LiBGRZPuxiMiDIpIrIptFZLLLsdeIyC775xp/vL6vKmsbqKhtYHB8VG9nRSmleoTPK3mJSAYwBzjgknwBMNr+mQY8AkwTkYHA3UAOYIANIrLMGHPc13z44mh5LQCpCVryV0oFB3+U/P8E3IEVzB3mA88Yy1ogSUSGAucDq4wxx+yAvwqY64c8+KSgrAZAS/5KqaDhU/AXkflAvjFmU6tdacBBl8d5dpqndHfnXiAi60VkfWFhoS/Z7JCj5D9YS/5KqSDRYbWPiKwGhrjZtRj4OVaVj98ZYx4DHgPIycnp1qXVj9ol/1Qt+SulgkSHwd8YM9tduohMBLKBTWINjkoHPheRqUA+kOFyeLqdlg+c2yr9vS7k268KymqICAshIdrnJhCllOoXulztY4z50hgz2BiTZYzJwqrCmWyMOQIsA662e/1MB0qNMYeBlcAcERkgIgOw7hpW+n4ZvskvqSY9KRrREb5KqSDRXUXdFcA8IBeoAq4DMMYcE5ElwGf2cb82xhzrpjx4Lf94NWkDons7G0op1WP8Fvzt0r9j2wALPRy3FFjqr9f1h7zj1cwZltjb2VBKqR4T9CN8q+saKa6sI11L/kqpIBL0wd/Rx39oovb0UUoFj6AP/kcc3TwTNPgrpYJH0Af/Amfw1wFeSqngEfTB/2iZY14fLfkrpYJH0Af/I2U1xESEEhepA7yUUsEj6IP/4dJqhiRG6QAvpVRQ0eBfWsMQrfJRSgWZoA/+xRV1uoKXUiroBH3wL6qoZVCsBn+lVHAJ6uBfVddAVV0jg+IiejsrSinVowI6+FfVNfDn1TvZeLDE7f7iijoAUuK05K+UCi4BHfyr6xr58+pdbM7zEPwrreCvJX+lVLAJ6ODvYDysA1ZkL9+YrCV/pVSQCejg7+i7bzxE/+JKK/hryV8pFWwCO/h3sL/IrvPXkr9SKtgEdPB38LT6e1FFLXGRYUSFh/ZofpRSqrcFdPDvaMaG4oo6rfJRSgWlgA7+Dp4afAvKakiN16kdlFLBJ6CDv9i1/p6qfY6U1ZCqK3gppYJQQAf/9lp8jTEcKa1hiC7iopQKQoEd/G3uunrWNjRR29BEUozW+Sulgo9PwV9E7hGRfBHZaP/Mc9l3p4jkisgOETnfJX2unZYrIot8ef2O8+d5X1lNPQAJ0eHdmQWllOqT/LF81Z+MMQ+4JojIeOAK4ERgGLBaRMbYux8GzgPygM9EZJkxZpsf8tEpZdUNACRE6QpeSqng012Rbz7wgjGmFtgrIrnAVHtfrjFmD4CIvGAf2y3B31Hwd9fbx1nyj9KSv1Iq+Pgj+N8sIlcD64EfG2OOA2nAWpdj8uw0gIOt0qe5O6mILAAW2A8rRGRHVzO44HckL4Aid/tm/q6rZ+0VyXi4jn5Ir6VvCpRrCZTrAN+uZbinHR0GfxFZDQxxs2sx8AiwBKs35RLgf4Hvdi2PLRljHgMe88e5RGS9MSbHH+fqTYFyHaDX0lcFyrUEynVA911Lh8HfGDPbmxOJyOPAG/bDfCDDZXe6nUY76UoppXqIr719hro8vBTYYm8vA64QkUgRyQZGA+uAz4DRIpItIhFYjcLLfMmDUkqpzvO1zv/3IjIJq9pnH/B9AGPMVhF5CashtwFYaIxpBBCRm4GVQCiw1Biz1cc8eMMv1Ud9QKBcB+i19FWBci2Bch3QTdcinua6V0opFbiCYoSvUkqpljT4K6VUEAro4N+TU0n4i4jsE5Ev7eky1ttpA0VklYjssn8PsNNFRB60r2+ziEzu5bwvFZGjIrLFJa3TeReRa+zjd4nINX3kOvr0VCaeiEiGiLwrIttEZKuI/NBO74/vi6dr6VfvjYhEicg6EdlkX8ev7PRsEfnUztOLdqcY7I4zL9rpn4pIVkfX5xVjTED+YDUo7wZGABHAJmB8b+fLi3zvA5Jbpf0eWGRvLwJ+Z2/PA97EGsw8Hfi0l/N+NjAZ2NLVvAMDgT327wH29oA+cB33AD9xc+x4+7MVCWTbn7nQvvL5A4YCk+3teGCnnef++L54upZ+9d7Yf9s4ezsc+NT+W78EXGGnPwrcZG//AHjU3r4CeLG96/M2H4Fc8p+KPZWEMaYOcEwl0R/NB562t58GLnFJf8ZY1gJJrbrf9ihjzAfAsVbJnc37+cAqY8wxY40WXwXM7f7cN/NwHZ44pzIxxuwFHFOZ9InPnzHmsDHmc3u7HNiONdq+P74vnq7Fkz753th/2wr7Ybj9Y4CZwMt2euv3xPFevQzMEhHB8/V5JZCDfxptp5Jo74PSVxjgbRHZINYUFwCpxpjD9vYRINXe7g/X2Nm89+VrutmuClnqqCahH12HXV1wClZJs1+/L62uBfrZeyMioSKyETiK9UW6GygxxjS4yZMzv/b+UmAQPl5HIAf//upMY8xk4AJgoYic7brTWPd7/bJ/bn/OO9ZUJiOBScBhrKlM+g0RiQP+DfzIGFPmuq+/vS9urqXfvTfGmEZjzCSsWQ6mAuN6Og+BHPzbm2KizzLG5Nu/jwKvYn0wChzVOfbvo/bh/eEaO5v3PnlNxpgC+x+2CXic5tvrPn8dIhKOFSyfM8a8Yif3y/fF3bX05/fGGFMCvAuchlXF5hh465onZ37t/YlAMT5eRyAH/343lYSIxIpIvGMbmIM1ZcYywNG74hrgNXt7GXC13UNjOlDqcivfV3Q27yuBOSIywL59n2On9Srpp1OZ2HXDTwLbjTF/dNnV794XT9fS394bEUkRkSR7OxprfZPtWF8C37QPa/2eON6rbwL/te/WPF2fd3qqhbs3frB6LuzEqk9b3Nv58SK/I7Ba7zcBWx15xqrfewfYBawGBprmXgMP29f3JZDTy/l/Huu2ux6r/vH6ruQda2bYXPvnuj5yHf+w87nZ/qcb6nL8Yvs6dgAX9KXPH3AmVpXOZmCj/TOvn74vnq6lX703wEnAF3Z+twC/tNNHYAXvXOBfQKSdHmU/zrX3j+jo+rz50ekdlFIqCAVytY9SSikPNPgrpVQQ0uCvlFJBSIO/UkoFIQ3+SikVhDT4K6VUENLgr5RSQej/AVd5Ku6AzkG0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.ylim(-500,200)\n",
    "plt.plot(mean_rewards)\n",
    "display(best_reward)\n",
    "display(np.sum(episode_lengths))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 287
    },
    "executionInfo": {
     "elapsed": 2233,
     "status": "ok",
     "timestamp": 1603342065461,
     "user": {
      "displayName": "Nimish S",
      "photoUrl": "",
      "userId": "16860995465865824316"
     },
     "user_tz": -330
    },
    "id": "Ojx0TJQbob39",
    "outputId": "d626a1df-fef5-4477-8263-d5f3e231b32d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fd1feada208>]"
      ]
     },
     "execution_count": 25,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5wU1Z338c9vZpiB4TrDdZhhuI4gAsIwAgKKF+SmCd4XTZQYXXTF20aT4C3Ga0yyyT7Ja13zsLtuNPGJursaiZoYNMlmTR5UNF64hDjxhgQVF0VdVATO/tHVQ09PX6eruqu7v+/Xa15Tfaq66lRX969OnXPqlDnnEBGR8lJR6AyIiEj+KfiLiJQhBX8RkTKk4C8iUoYU/EVEypCCv4hIGco5+JvZCDP7tZltMrONZnapl15vZmvN7CXvf52Xbmb2fTNrN7MXzKw11zyIiEh2/Cj57wUud85NBGYBK81sIrAKeNw51wI87r0GWAy0eH8rgNt9yIOIiGQh5+DvnNvunHvWm/4A2Aw0AkuBO73F7gRO9KaXAne5iHXAADNryDUfIiKSuSo/V2Zmo4BpwJPAUOfcdm/Wm8BQb7oR2Brztje8tO0xaZjZCiJXBvTu3Xv6hAkT/MxqShv+sgvn4JDh/agw48VtuzrmjairZeu7uwGY2NCPnbv38Oauj7080+l9QKf3xutbU8UHn+wNZB8mN/Zn737H5u3vB7L+WI0DerHtvY8C3045O2R4Pzb+JfhjKZmbMKwff3wz+GPSo7KCCcP6duu9zzzzzDvOucGJ5vkW/M2sD/AfwGXOuffNC34AzjlnZlmNI+GcWw2sBmhra3Pr16/3K6tpTbj253z86X6euGEhtdVVjFr1cMe87yybyqX3PAfAN8+azoofPUP0sqVnjwo+/nQ/v7thEb2qKwE6vTfe52c18+N1rweyD+tvPZ4dH3zCYTc/Fsj6Yw2v68X+dxX8g/Rf1y/kkOseLXQ2JMajVx7LrG88Hvh2mup68cRXj+nWe83stWTzfOntY2Y9iAT+u51z93vJb0Wrc7z/b3vp24ARMW9v8tKKzp59+3N6f2XMCdJv//q7VwJbdzwNDxU8fcTlK6jflx+9fQz4F2Czc+67MbPWAMu96eXAgzHpZ3u9fmYBu2Kqh8pKkD/o63+2KcC1i4gr8lOyH9U+c4CzgBfN7Dkv7SrgVuA+MzsXeA043Zv3CLAEaAd2A+f4kAdfGZESuUq0mdHIsMHTZ1y+gqogyDn4O+eeAJJl79gEyztgZa7bDcLaTW8xdnDvjtfrXv5vfvj7Vzst8+k+/Qgl//StK19Bnfd97e1TzPbtd/z1XesZ3r9nR9q5d3ZtZL7i357PaH1fX7Oxy4kjngpzIsWr2H+/Cv6effsjR/IvXrfNTHz4cfJumukCfz4Ue52kHFDsgUbCR2P7eLoTKG94SI2qoCqJvNCHLD5T8Pd0p2S1e8++3LYZ8C/akjbF+Gu/iqVShor9W6/gLznL10mmnKkKT/ym4C85C/BeNfHo4kr8puDvKcSPK+htqrRYOnQkw6fY771Q8Pf4ESiL/LsgImVEwb+EqS6+dBR7KVPCR8Hf48dvK2x136r2KR06kuFT7OdjBX9PIY5jkX93RCQPgrrqU/AvYar2KR3FXsqU8FHw9/hxdtUPVIKiKjzxm4K/pyDVPvo9i0gaQYUJBX8flWuDb8h2uzSpoCA+U/D3qBTeffrogqfPWPym4B/lw68rbCcQNfiKBCdfv/fQPsO3VPhZRZL5uoL99qjap3SErWAhxU/B30dhq/OX0qHePuUrqGOv4O/xo2T18af7c1+Jn/J1WZqfzZQ1lfzDp9hPyAr+nlLs6rk/TzulCx6R4qPgX8LyVTIp7vJPcdBnXL5C3eBrZneY2dtmtiEmrd7M1prZS97/Oi/dzOz7ZtZuZi+YWasfeciVn+NnhOUSPSz5kNxpVM/wKfZD4lfJ/4fAori0VcDjzrkW4HHvNcBioMX7WwHc7lMeclLkxzGhfO1Tsf8IRMIsqI4kvgR/59xvgZ1xyUuBO73pO4ETY9LvchHrgAFm1uBHPopN4E/yylNULvaGr2KgE2z5CnW1TxJDnXPbvek3gaHedCOwNWa5N7y0TsxshZmtN7P1O3bsCDCbEaX44yr2m1BEwixvV9YBrTcvDb4uUgTNah+cc6udc23OubbBgwcHlLOY7ZVg6TVvwT8/mxERHwUZ/N+KVud4/9/20rcBI2KWa/LSCsvHCBaWYFiKJ7Rypasr8VuQwX8NsNybXg48GJN+ttfrZxawK6Z6qKwEHZzz1c9fgSl4OpGXr6B+X1V+rMTMfgIcBQwyszeA64BbgfvM7FzgNeB0b/FHgCVAO7AbOMePPOSqFG/yyl/3QAUmKT/F/vvyJfg7585IMuvYBMs6YKUf2/VTIUqvQY8FpJBcOnR1JX7THb4FVColfwWm4OkjFr8p+Ht8HdI5JNEwJNkQH4TlOyUHFPtNlAr+nlL8beWtwTc/mxERHyn4F1DQQTNvA7uV4pkzZPQJi98U/D2l+ONSTC4dOpbhU+w3USr4e3wd1dO3NeWm2L+cIhIcBf8CCry3T57C8sj62rxsp7zpFFuu5h0UzPA2Cv6eUrysztc+TRzeLz8bKmOl+P0sfvk5KJfNbwlkvQr+JUyjeooUP3X1DFghAljQ1TL56+2Tl82UNX3E4jcFf2D3nr1c8+CG9AtmKCzBMF/9/Ddu35WfDZWxsHyn5IBvP7olL9sJ9ZO8it0Pf/8qv/1T8A+Mybd89b/fsO39vGynnO3es7fQWZA4j258q9BZyImCP7A/X0XkeAFv9q33Pw52A5I3l9zzh0JnQQpEdf4BKtVL6gt+/GyhsyA+2brzo0JnQUqMgr+ISIipzr+YlOiVhIjkn6p9isgftr6b0XI6R4hIoSj4438Q/odftfu8RhERfyn4ByDoxzOKSPlQnb+ISBlSnX8J0kNQRKRQFPwp3X7+IlL8Sq7ax8wWmdkWM2s3s1WFykch/fS5vxQ6CyJSpgoS/M2sErgNWAxMBM4ws4mFyIuISJiVWp3/DKDdOfeyc24PcA+wNN+ZmHPrr/jRutfyvVkRkYIrVPBvBLbGvH7DS+tgZivMbL2Zrd+xw98RN3/X/g6/3Pgm2977iGt/uiFv496LiGQrqDr/qmBWmzvn3GpgNUBbW5uv0flz//ykn6srW63NA3j29fcKnQ0pgOqqCvbs3V/obEgOClXy3waMiHnd5KVJEbEQ38321NXHFjoLGRlQ2yMv26mq8O9YnTmzmX/9wmEZL7/y6LG+bVv8U6jg/zTQYmajzawaWAasKVBefGeENyj6Kcx7WSzH4IiWwXnZzqnTm3xb101LJ2W03OTG/jx/3QK+vHACM0bX+7Z98UdBgr9zbi9wEfAosBm4zzm3sRB5CcJTr+4sdBY6WTJ5GIsOGeb7evNZ8P/qoglZLZ9Nibq+d3W22fFNLh9hy5A+GS13/rwxLJ89KoctdVaR4iri+CkNHdO9ayrp3ytyHCpDfJUYhDNnNhc6C2kVrJ+/c+4R59xBzrmxzrmbC5WPRKorS+vet741PVh59LhCZ4O2kXXdfu8F88Z0en3751pTLt8ji2NYyLA0cXi/Tq8b+vfM+L1HHpTZVcOViw/m4IZ+/L/zZmaVt+44a9bIhOkVKQ7HoD5dT74/Pjf4vAZpcJ+aQmchrdKKcik8+Nw2HnwucbNCl360JVZIqaiAHlWF36lsTkDfWzaV0YN6AzCivldQWQISX8EcNqqOQ5v6c37cSefVW4/3dduHjxnY6XWvHpUZv3dflo8fnT1uUJe00+KqgyY39s9qnVHpqnXe/yj5M4jHD+vbJS3Iq7EL5gXfBrG/G53zj2jpenyCVDbB/9J7nuPSe57LaNlS68Vw3hFjGD+0b5dAE3X+kWO6pM0/eEiXtK9/ZiI3npi8vjddvfLRE4bw80uPSJPbiKVTG/n1FUex4fqFrP3beV0al/3tnNt53c31tfzbBbN58KK5XLn44JzWnK5qprqq609w7ODeadf7ndMOZfiAzK8Sou74Qlun17PivhP3Xzg763Vm4sVtu5LOO6W16/cm1ZVCrmqrMz/BZiLRb6I7wf9Heb7aKZvgX8761FRhZiyfnfiSPFNfmDOas2aN7Og5Et+omkmHkoMb+qVfKEafmip6JigNRzc1PYeqpKgj40pcowalDr5nzMi8Pvevj+h6Yo1aOnU4zfW1ndImDu9HU11tkncccMr0Js6dO4ZRA9MvG+uYCUM7Xb3Efn5fmD2KHpUVCQsD3TEwpuoj1XEa2i9yEquM+QJVFFEbQaKqrn0Zlh9njSlcQ7iCfxnJppYgVQPrT1fOAbrWV9eluFTvUXngx/yry+d1KYF2V6JS3HETh2b03s/Paub56xawdFqn+wu7nMSevfY4AP6qLdI7eUpT6qqRk2LWl+gGwhOnDmf9NfP53rJpHQGvpqqC+y+czbdPPbRTEEylssKYf3Bm+5pM355db/W5csnBDPSh2uUbJ0/umJ4U9125cnHk+3XtCROZM24QP/j8dL6ycDwQKSAEGfzjj98jlxyR8oq2O6Il/2MmdL2CjtWnJv2tVqU2vEOoFOP9vU9edSxnzmxO2FiWTLJL0USpLUO71sNGTWrsz/0XzuaqJZ2rROal6LY4pO+BKooxg/tw9PiuP4rYYJHIk1cd6Luf6pid6ZXMoyX0RLF00w0Luf6zk+jfq0eXobXjF6/vXc2rtx7PN0+dknB+vCu8IAbJT7iD4hoEnYPW5jp6VVf6EvgmJKhHTyS2Oi3RZr/l7XN39Ot5oMdVfAN8Y10vXr31eM6dOxqARZOGUeUtM3N0PWMH9+aSY8bxn18+KuvtPnTx3JTzjxo/hMe+NK/jdVWlJW2o7o7zjxzT0R6TqKr1d6uO6ZhetThxISubq8vuUvAvUkP79eSWkyaz/prjMn5PbCB6IMe63dbmuq711Sli1sK4rqaJbhBLVxc7tF9PeqdZZstNizjaK23d7JXmpjV3rXKora5KWsJOd/NaNrE5uk+fPXR4wvnRQB97heBHZ7NfXHZkRsv1rkn8eUb3MdFJOt5NJ07iM14Xz5FJqqGqMtip6Em4wgwz40sLxjNyYOoquJdvWcK9K2Z1nEQgUjhJZ0i/3HvjjBxYy6YbFnZKe+DC2axaPIHzjhhNa/MATm5t7PK+xgEHOjCMG9KXL84Z3XFVeWCZA4WlkhvSWfIv+uM6YUpDwoCY+wa6JrU2D+DZa4/j6uPTN5yaGcsPH8n3lk1Nuky0pBTtDTK4b+cfcY+YlsKKCuNnF83lX8/J/G5U8Lez16TG/nxv2VRuibmqiT25RM8/sSfmxZMO9JWPdeLUrieQlqFdG5SzadCsqarkqiWRzzS2DSd6ZZJJFdRpbU18ftZItty0iIb+iXtmxQew3gmqO06a1sihTf0574jRXeYlU1FhzBwzkGtPSD4ocH3vap746tEZrzNTdbXV1FZ33o9pzXWYGQ39e3H/hXMY2KeGZYeNSHpSBPjaZyZ2XFU21fXi7MNHpqxC9Utox/YR/8T/0IKqT01UwzF2cJ+k3fZ+8tezePrVnXx37Z860q5Pc/foWYeP4qzDR+Gc49unTmHJ5AbufzbShfcXlx3R5QakyWnq5xPl+4RDEwffqGyHtVg6tWvpL6qywvjinNGdtnnitEYuuzfSM+37Z0zjkp/8ATjQMBrr9LYRTBjWj6W3/Q6ItKeMGZzZzV9Rib4PPzxnBv/5p7eTHrvDRtVz/JQGrlgwnpqqyMkm+j/xNg5Mf+vUKRyV4B6FgX1qePCi5FU2154wkRsf2pR0/t/OP4gdH37cJX3eQYO7NKJn+wt44qtH897uT/mfT/YyelBvZtzyOMdPTv09ibr1lEhgH7Xq4Y60e1fM6lJwiWwnUiW0f7/j6gc2ZJnL7Cj4Q0k/yquywjoalaJ1/tHfet+eVXzwcfL+19mKvZzNxOFjB3L42IFsefMDHn5xe1Y/SDPjtLhL5f3d7aEbd/inNA1Ive0sVp1o2QUxDdJmxtc+k7zU+tlDh/Pjda/x1CuJ7xo3Mw4dcSC/sYH/4Uvmsnn7B1nk9oBh/XvyV4clr3eurqrgtjNT32gX66xZo7jt138GIies7lgwcSjD+/fkmdfe5Z+feKXL/EvntyR8300JGnMTncC33LSI8df8Aog07s9tGcTF3om3qa6WppiL5U03LMzqngyAx750JFvf/QiAmUm6XUdVVBjN9bW8vnN3VtvIhoJ/EYrtOZNOXe2Bklv0HBct6V109Di+8fM/emnZ9QaKOrihH1vefJ91Vx3bqVG3O3K9IMlkaO6fXTSXbe99lHKZdEMRpKsyS/Zs5s/PaubH615ncYYlxqhoo36qRvhEDhnen0OGZ3Dl42U3k8//9Lb0YwStPms6g+JKtcP69+T8I8cwL8O7kmNdfMw4/um/XmZov56MqK9l8eQGhvSrSXkn/n/8zWxOuf33QOIqpkRtRzVVlTz2pSN558M9Hfc/PPTCX1iW4CQYX92TiXFD+jJuSHbHMEgK/iF22fwWTp3exIq7nmHT9vc70rNp5I39Qe+P+5HHhqjJTQN4fmv2wzM/cslcb52Rlb7w9QU8tuktrv/ZJnZ99GlGAaWpLnLFEHui6o5MLuAmN/XvUhUUf9JI1P0x1rgMx9SBziXMm06czE0npu7RlMg3Tp7CnHGDOKW1kWnNA9LmL1sLDxnGzY9s5rQ0gf3Mmc3cclL6/C9IMo7UlUu6d8Pc5QvGc/mC8Z3SVhyZ+i7d6SPreOnmxezesy/h/GRVd5EAfeD1/z3Lny7JYaTgH0J/uPY49u53HXWC8d/T6GBZmUj0FU9Ux9vdQnf8j6hfzx6c3NrEyIG9OeX232d0E9blC8YzrbmOOQmGH8jEuCF9aH/7w27X3sW/b2DIxmXp36sHn5sZ6Yo4Nsv6/GROntbIlrciVULNA2szGrbipGnJ2y7CqEdlBf17FW+fluOnNHD7b/7MgF7BNP4q+BO+fv5BtfSfMKWBZ17byRVeKSrIeyinj6zjia8enVE7QHVVBYsmdX/U0WWHjeCmhzfT0I3hDrrr/gtn8+Ibu7huTTgGo33o4rns+OCTjJf/7l8l71GVzGGjNCxzPn15wXgumDc2q8JeNhT8i0BsyXTNRXO6zP+70w7l6Vd2cu/6rV3mxerZo5JvnNz9m3aylckwBX44d+5ozj58VMJxcqJ+v+oYdu/xr3G7tbmO1ua6tME/X4MUZNK3vdz95oqj+B8fvwNBq6iwwAI/qJ9/0UnUE+XU6U2cflikvja+MThVnfvhYyONWkeNz88DRYJiZikDP8DwAb2SNrYF2dmriIaoKXmjBvXOqAG8O86LucmsWKjkXwQyiU3RADa5sT9XH38wp9z+/4HUT7Sa0jSgo67XbEuu2SxaYav2C5sHV87hlXf+p9DZCLVrTpjINSluNAsjlfwJfzf/ZF0HOy3j/TezTiNnZlryvHrJwYxJM5plqcrk8y0EP5+7m4tDRwzgxCJr7M1GTZqrxlKlkn+JMbr3/Nq2UfX86oqjOt2FKLnr7rOEn7zq2JJ7olxYpRs/qFQp+Jeg2NJ+OMqO5au7df6JhnIQ8ZOCfwI9Ko1P9xWmKmBBgrHoM6mVSLbM351+aLfzMmZwb17eUfp1veGs9JGg3X3eTNrf/rDQ2SgYXVfS9Q7Pc+YUruV+fqLgn0F4ij7R6aTWxk6lzdljs7txKrbXwsMXH8F/fcX/0RDDJqRV/hKwOeMGsXz2qEJno2BU8k8gbN3zMhlzZ0i/nvz5liVUGDldtcT2WuhVXcmA2uD6GZcqnUykGORU8jez08xso5ntN7O2uHlXmlm7mW0xs4Ux6Yu8tHYzW5XL9oNSiOeHJnqIdVSmD4OurLCshxsWCLLiR4dDwirXap8NwMnAb2MTzWwisAw4BFgE/KOZVZpZJXAbsBiYCJzhLVtQ8bfFh6SHXYf9WQ63qYCTnWBv8tLBkHDKqdrHObcZEn7BlwL3OOc+AV4xs3Zghjev3Tn3sve+e7xlkz+hIQ/uW/9Gp9fphvT122XzW9i6M/kww/uyjE4KNyKSTlANvo1A7EAzb3hpydJDJf5pUEFLN5pltg8pUWkzOzPHDMz6wRwixS5t8Dezx8xsQ4K/pUFmzMxWmNl6M1u/Y8eOIDfVddsFKDuniteZ1vl3rCvHvJSb+t7VbL5xUSDr1rGQsEpb7eOcm9+N9W4DYp/V1uSlkSI9frurgdUAbW1tee0/kajgX1tdmfTBEEHLOvj7GHF0FSFSmoKq9lkDLDOzGjMbDbQATwFPAy1mNtrMqok0Cq8JKA/dlqjaJ8gQmC6271O1T9HSoZCwyrWr50lm9gZwOPCwmT0K4JzbCNxHpCH3F8BK59w+59xe4CLgUWAzcJ+3bKiE7Qe7+uzphc6CiJSYXHv7PAA8kGTezcDNCdIfAR7JZbtBS1Tnf9Cwvvzh9eyfcZuJdKNKtqZ5YLiEVyHaj0QyoeEdMvQvyw8rdBZERHyj4J9AfLXPxIZ+gT5OTYrf3CTddcNWhSgSpeCfgcqKYC/e1UBb/JLdG6IjK2Gl4J+BoG/6Cmr1PXvo8BbC8AG9Cp0FkbQ0qmcC8e2vlRbsCaBTwd+nOxq+e/qhTFNDcV798JzDaK6vpbLCGNSnhnc+/CT9m0QKREXDJPrWHDgv5jrK5+ltyUfsjPD/xHJyaxOjy/SZvIVy1PghjBncx3vlncVV7yMhpeCfQHysj5b6Z48d2K311VSlHjcmzFX+Ic6aiORAwT8D0VE+RwVUklaAFZF8K7vgn+6GqqhFk4Z1TFfEfUrnxjzqcKT3+MRcFOLhMRIs11Hro2Mr4VR2wf+an27IaLlbTp7Md06LPPw8PjjH1qX/6Iszc86TYn/piRYxdGwlrMou+N/95OsZLdejsoL63tVA1+Afe+3QPLCWp646tlt5OWNGMwCNsV0DFSxKig6nhFXZBf9MRC/Z+/aM9PgZUZ+63/aQfj27tZ1bTprEn29ZwsA+NTEb79aqJGS+dcoUxg/tqzvDJbTUzz+FtlH1/OPnWjlmwpBO6dmW5qIXDnW1PTh+SgM/Xve6l25UevP2ec/pzfdTxKSzfj39+UnMnziU+ROH+rIukSCo5J9AbC3PkskN9Ix7xF8mhfO7z+vaFnDZ/IO46cTJCZf/ZG/kQTG6K7dw7l0xi1/+7bys3nPVkgkB5UYkWCr5ZyGbMvnQfjXpF4qxZ2/kiS3p7gmQ4Mwck/19HC1D+waQE5HgqZjpk/abF3d6naj7ZqpupufPG0ttdSVtI8M1JIN6q4iUJgX/LJwyPTJMwxHjBjFhWOcSX1VlBb2rD5TaKyuMH3y+lUuObclo3YeNqmfTDYuo83oYSXHROVKKjap9stDaXMertx4PwM8untvRSBvVMrQvz22NPO3LMBZNamDRpAa+9mBm9xZI8VInLSk2Cv7d1KOygrh24E5VJLHTvbwrgmrV54tISCj4+yi2Sj82+F96bAs1VZWclnZ0TylWqvaRYqPgH5DYp3PVVlfxpeMOKmBuREQ6U4Ovj2JL+7pXS0TCLKfgb2bfNrM/mtkLZvaAmQ2ImXelmbWb2RYzWxiTvshLazezVblsP8w0mmN5UYOvFJtcS/5rgUnOuSnAn4ArAcxsIrAMOARYBPyjmVWaWSVwG7AYmAic4S1bclTyF5Ewy6nO3zn3y5iX64BTvemlwD3OuU+AV8ysHZjhzWt3zr0MYGb3eMtuyiUfoZQi+E9rHsDBDf3ylxcJnM71Umz8rPP/IvBzb7oR2Boz7w0vLVl6Sbg05oauVNU+D1w4h1tOSjzGT9io+kqkNKUt+ZvZY8CwBLOuds496C1zNbAXuNuvjJnZCmAFQHNzs1+rDdSccYM6plXtIyJhljb4O+fmp5pvZl8ATgCOdQcGr9kGjIhZrMlLI0V6/HZXA6sB2traiqI9LTbemwbFEZEQy7W3zyLgK8BnnXO7Y2atAZaZWY2ZjQZagKeAp4EWMxttZtVEGoXX5JKHsFLoLw8HD4u03ZzeNiLNkiLhkutNXv8A1ABrvZLuOufcBc65jWZ2H5GG3L3ASufcPgAzuwh4FKgE7nDObcwxD6ERW9rXQ9nLw7D+PTvGexIpJrn29hmXYt7NwM0J0h8BHsllu0XB59j//NcW+LtCESlrGt4hge421nau8/clKx361+pZsCLiHw3vkEB3q2w6D++gah8RCS8F/4Ao9ItImCn4J9D9kr/FTPuVm8Iqlf0Qkc4U/BPw4wYtVfuISJgp+CdQodtzRaTEKfgn4EfoV8lfRMJMwT8BP4ZmUOwXkTBT8E/Aj1K7Yr+IhJmCfwJ+lNpV7SMiYabgD1RXdf4Y/GjvVewXkTBT8E+gf6/qnNehIZ1FJMwU/KHL07cXHjK0MPkQEckTBX/AxUV/ldpFpNQp+AOuKJ4TJiLiHwV/YPywvoXOgohIXin4A0P61hQ6CyIieaXgLyJShhT8UQOviJQfBX8RkTJU8s/wfefDT/j4032FzoaISKiUfPBvu+mxtMuo0kdEyo2qfSQlNYeIlKacgr+Z3WhmL5jZc2b2SzMb7qWbmX3fzNq9+a0x71luZi95f8tz3YGw+Yczp3Hi1OGFzoaISEq5lvy/7Zyb4pybCjwEfM1LXwy0eH8rgNsBzKweuA6YCcwArjOzuhzzkDM/S7cnTBnO/1k2zb8ViogEIKfg75x7P+Zlbw4MkbYUuMtFrAMGmFkDsBBY65zb6Zx7F1gLLMolDyIikr2cG3zN7GbgbGAXcLSX3AhsjVnsDS8tWXqi9a4gctVAc3NzrtkUEZEYaUv+ZvaYmW1I8LcUwDl3tXNuBHA3cJFfGXPOrXbOtTnn2gYPHuzXapNQq6aIlJe0JX/n3PwM13U38AiROv1twIiYeU1e2jbgqLj032S4fhER8UmuvX1aYl4uBf7oTa8BzvZ6/cwCdjnntgOPAgvMrM5r6F3gpRWUujOKSLnJtc7/VjMbD+wHXgMu8NIfAZYA7cBu4BwA59xOM7sReNpb7gbn3M4c8yAiIlnKKfg7505Jku6AlUnm3QHckct2RUQkN7rDFzX3pmL6dOMaFUsAAAh0SURBVERKkoK/iEgZUvAXESlDCv6ot4+IlB8F/zi/uOyIQmdBRCRwCv5xJgzrV+gsiIgETsEf9WgRkfKj4C8iUoYU/EVEypCCP+rtk4o+G5HSpOAvIlKGFPxFRMqQgj8wqbF/obMgIpJXZR/8H7p4Ln8zb2yhsyEiklc5P8O32KnULyLlqOxL/iIi5UjBX0SkDCn4i4iUIQV/EZEypOAvKekGX5HSpOAvIlKGFPxFRMqQL8HfzC43M2dmg7zXZmbfN7N2M3vBzFpjll1uZi95f8v92L6IiGQn55u8zGwEsAB4PSZ5MdDi/c0Ebgdmmlk9cB3QBjjgGTNb45x7N9d8iIhI5vwo+f898BUiwTxqKXCXi1gHDDCzBmAhsNY5t9ML+GuBRT7kQUREspBT8DezpcA259zzcbMaga0xr9/w0pKlJ1r3CjNbb2brd+zYkUs2RUQkTtpqHzN7DBiWYNbVwFVEqnx855xbDawGaGtrc2kWFxGRLKQN/s65+YnSzWwyMBp43iKPe2oCnjWzGcA2YETM4k1e2jbgqLj033Qj3yIikoNuV/s45150zg1xzo1yzo0iUoXT6px7E1gDnO31+pkF7HLObQceBRaYWZ2Z1RG5ang0992QoJie4yhSkoIa0vkRYAnQDuwGzgFwzu00sxuBp73lbnDO7QwoDyIikoRvwd8r/UenHbAyyXJ3AHf4tV0REcle2T/MJeq3Xz6aHR9+UuhsiIjkhYK/p3lgLc0DawudDRGRvNDYPiIiZaisg/8ZM5oLnQURkYIo6+DfeUQKEZHyUebBX0SkPCn4i4iUIQV/SUn394qUJgV/EZEypOAvIlKGFPxFRMpQSQf/3Xv2FjoLIiKhVNLB/6M9+wqdBRGRUCrp4C8iIokp+IuIlKGSDv56CpWISGIlHfwldzp/ipSmsg7+TuO6iUiZKuvgLyJSrko6+KvGQkQksZIO/iIikpiCv4hIGcop+JvZ181sm5k95/0tiZl3pZm1m9kWM1sYk77IS2s3s1W5bD99/lLPP62tKcjNi4iEVpUP6/h759zfxSaY2URgGXAIMBx4zMwO8mbfBhwHvAE8bWZrnHObfMhH1qaPrC/EZkVECs6P4J/IUuAe59wnwCtm1g7M8Oa1O+deBjCze7xlCxL8RUTKlR/B/yIzOxtYD1zunHsXaATWxSzzhpcGsDUufWailZrZCmCF9/JDM9uSQx4HAe902cY3c1hjYSTcj3zx+fMq6L74TPsSPqWyH5DbvoxMNiNt8Dezx4BhCWZdDdwO3Ag47/93gC92L4+dOedWA6v9WJeZrXfOtfmxrkIqlf0A7UtYlcq+lMp+QHD7kjb4O+fmZ7IiM/sn4CHv5TZgRMzsJi+NFOkiIpInufb2aYh5eRKwwZteAywzsxozGw20AE8BTwMtZjbazKqJNAqvySUPIiKSvVzr/L9lZlOJVPu8CpwP4JzbaGb3EWnI3QusdM7tAzCzi4BHgUrgDufcxhzzkAlfqo9CoFT2A7QvYVUq+1Iq+wEB7Ys5jW4mIlJ2dIeviEgZUvAXESlDJR388zmUhF/M7FUze9EbLmO9l1ZvZmvN7CXvf52Xbmb2fW//XjCz1gLn/Q4ze9vMNsSkZZ13M1vuLf+SmS0PyX6EeiiTZMxshJn92sw2mdlGM7vUSy/G45JsX4rq2JhZTzN7ysye9/bjei99tJk96eXpXq9TDF7HmXu99CfNbFS6/cuIc64k/4g0KP8ZGANUA88DEwudrwzy/SowKC7tW8Aqb3oV8E1vegnwcyKjV88Cnixw3o8EWoEN3c07UA+87P2v86brQrAfXweuSLDsRO+7VQOM9r5zlWH5/gENQKs33Rf4k5fnYjwuyfalqI6N99n28aZ7AE96n/V9wDIv/QfA33jTFwI/8KaXAfem2r9M81HKJf8ZeENJOOf2ANGhJIrRUuBOb/pO4MSY9LtcxDpgQFz327xyzv0W2BmXnG3eFwJrnXM7XeRu8bXAouBzf0CS/UimYygT59wrQHQok1B8/5xz251zz3rTHwCbidxtX4zHJdm+JBPKY+N9th96L3t4fw44Bvh3Lz3+mESP1b8Dx5qZkXz/MlLKwb+RrkNJpPqihIUDfmlmz1hkiAuAoc657d70m8BQb7oY9jHbvId5ny7yqkLuiFaTUET74VUXTCNS0izq4xK3L1Bkx8bMKs3sOeBtIifSPwPvOef2JshTR369+buAgeS4H6Uc/IvVXOdcK7AYWGlmR8bOdJHrvaLsn1vMeScylMlYYCqwnchQJkXDzPoA/wFc5px7P3ZesR2XBPtSdMfGObfPOTeVyCgHM4AJ+c5DKQf/VENMhJZzbpv3/23gASJfjLei1Tne/7e9xYthH7PNeyj3yTn3lveD3Q/8Ewcur0O/H2bWg0iwvNs5d7+XXJTHJdG+FPOxcc69B/waOJxIFVv0xtvYPHXk15vfH/hvctyPUg7+RTeUhJn1NrO+0WlgAZEhM9YA0d4Vy4EHvek1wNleD41ZwK6YS/mwyDbvjwILzKzOu3xf4KUVlBXpUCZe3fC/AJudc9+NmVV0xyXZvhTbsTGzwWY2wJvuReT5JpuJnARO9RaLPybRY3Uq8Cvvai3Z/mUmXy3chfgj0nPhT0Tq064udH4yyO8YIq33zwMbo3kmUr/3OPAS8BhQ7w70GrjN278XgbYC5/8nRC67PyVS/3hud/JOZGTYdu/vnJDsx4+8fL7g/egaYpa/2tuPLcDiMH3/gLlEqnReAJ7z/pYU6XFJti9FdWyAKcAfvPxuAL7mpY8hErzbgX8Darz0nt7rdm/+mHT7l8mfhncQESlDpVztIyIiSSj4i4iUIQV/EZEypOAvIlKGFPxFRMqQgr+ISBlS8BcRKUP/C9P1Gbnj3GwwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.ylim(-500,200)\n",
    "plt.plot(episode_rewards)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyPTS6S64J1MKSUgohlUY6Rj",
   "collapsed_sections": [
    "M-bHj8PV3Cir",
    "piqynVGt3DQu"
   ],
   "name": "A2C.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}