8.gail.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.distributions import Normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Use CUDA</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cuda = torch.cuda.is_available()\n",
    "device   = torch.device(\"cuda\" if use_cuda else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Create Environments</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from common.multiprocessing_env import SubprocVecEnv\n",
    "\n",
    "num_envs = 16\n",
    "env_name = \"Pendulum-v0\"\n",
    "\n",
    "def make_env():\n",
    "    def _thunk():\n",
    "        env = gym.make(env_name)\n",
    "        return env\n",
    "\n",
    "    return _thunk\n",
    "\n",
    "envs = [make_env() for i in range(num_envs)]\n",
    "envs = SubprocVecEnv(envs)\n",
    "\n",
    "env = gym.make(env_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Neural Network</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_weights(m):\n",
    "    if isinstance(m, nn.Linear):\n",
    "        nn.init.normal_(m.weight, mean=0., std=0.1)\n",
    "        nn.init.constant_(m.bias, 0.1)\n",
    "        \n",
    "\n",
    "class ActorCritic(nn.Module):\n",
    "    def __init__(self, num_inputs, num_outputs, hidden_size, std=0.0):\n",
    "        super(ActorCritic, self).__init__()\n",
    "        \n",
    "        self.critic = nn.Sequential(\n",
    "            nn.Linear(num_inputs, hidden_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_size, 1)\n",
    "        )\n",
    "        \n",
    "        self.actor = nn.Sequential(\n",
    "            nn.Linear(num_inputs, hidden_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_size, num_outputs),\n",
    "        )\n",
    "        self.log_std = nn.Parameter(torch.ones(1, num_outputs) * std)\n",
    "        \n",
    "        self.apply(init_weights)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        value = self.critic(x)\n",
    "        mu    = self.actor(x)\n",
    "        std   = self.log_std.exp().expand_as(mu)\n",
    "        dist  = Normal(mu, std)\n",
    "        return dist, value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(frame_idx, rewards):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s' % (frame_idx, rewards[-1]))\n",
    "    plt.plot(rewards)\n",
    "    plt.show()\n",
    "    \n",
    "def test_env(vis=False):\n",
    "    state = env.reset()\n",
    "    if vis: env.render()\n",
    "    done = False\n",
    "    total_reward = 0\n",
    "    while not done:\n",
    "        state = torch.FloatTensor(state).unsqueeze(0).to(device)\n",
    "        dist, _ = model(state)\n",
    "        next_state, reward, done, _ = env.step(dist.sample().cpu().numpy()[0])\n",
    "        state = next_state\n",
    "        if vis: env.render()\n",
    "        total_reward += reward\n",
    "    return total_reward"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>GAE</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_gae(next_value, rewards, masks, values, gamma=0.99, tau=0.95):\n",
    "    values = values + [next_value]\n",
    "    gae = 0\n",
    "    returns = []\n",
    "    for step in reversed(range(len(rewards))):\n",
    "        delta = rewards[step] + gamma * values[step + 1] * masks[step] - values[step]\n",
    "        gae = delta + gamma * tau * masks[step] * gae\n",
    "        returns.insert(0, gae + values[step])\n",
    "    return returns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>PPO</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ppo_iter(mini_batch_size, states, actions, log_probs, returns, advantage):\n",
    "    batch_size = states.size(0)\n",
    "    for _ in range(batch_size // mini_batch_size):\n",
    "        rand_ids = np.random.randint(0, batch_size, mini_batch_size)\n",
    "        yield states[rand_ids, :], actions[rand_ids, :], log_probs[rand_ids, :], returns[rand_ids, :], advantage[rand_ids, :]\n",
    "        \n",
    "        \n",
    "\n",
    "def ppo_update(ppo_epochs, mini_batch_size, states, actions, log_probs, returns, advantages, clip_param=0.2):\n",
    "    for _ in range(ppo_epochs):\n",
    "        for state, action, old_log_probs, return_, advantage in ppo_iter(mini_batch_size, states, actions, log_probs, returns, advantages):\n",
    "            dist, value = model(state)\n",
    "            entropy = dist.entropy().mean()\n",
    "            new_log_probs = dist.log_prob(action)\n",
    "\n",
    "            ratio = (new_log_probs - old_log_probs).exp()\n",
    "            surr1 = ratio * advantage\n",
    "            surr2 = torch.clamp(ratio, 1.0 - clip_param, 1.0 + clip_param) * advantage\n",
    "\n",
    "            actor_loss  = - torch.min(surr1, surr2).mean()\n",
    "            critic_loss = (return_ - value).pow(2).mean()\n",
    "\n",
    "            loss = 0.5 * critic_loss + actor_loss - 0.001 * entropy\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Loading expert trajectories from №3 notebook</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    expert_traj = np.load(\"expert_traj.npy\")\n",
    "except:\n",
    "    print(\"Train, generate and save expert trajectories in notebook №3\")\n",
    "    assert False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Generative Adversarial Imitation Learning</h1>\n",
    "<h2><a href=\"https://arxiv.org/abs/1606.03476\">Arxiv</a></h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Discriminator(nn.Module):\n",
    "    def __init__(self, num_inputs, hidden_size):\n",
    "        super(Discriminator, self).__init__()\n",
    "        \n",
    "        self.linear1   = nn.Linear(num_inputs, hidden_size)\n",
    "        self.linear2   = nn.Linear(hidden_size, hidden_size)\n",
    "        self.linear3   = nn.Linear(hidden_size, 1)\n",
    "        self.linear3.weight.data.mul_(0.1)\n",
    "        self.linear3.bias.data.mul_(0.0)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = F.tanh(self.linear1(x))\n",
    "        x = F.tanh(self.linear2(x))\n",
    "        prob = F.sigmoid(self.linear3(x))\n",
    "        return prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expert_reward(state, action):\n",
    "    state = state.cpu().numpy()\n",
    "    state_action = torch.FloatTensor(np.concatenate([state, action], 1)).to(device)\n",
    "    return -np.log(discriminator(state_action).cpu().data.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_inputs  = envs.observation_space.shape[0]\n",
    "num_outputs = envs.action_space.shape[0]\n",
    "\n",
    "\n",
    "#Hyper params:\n",
    "a2c_hidden_size      = 256\n",
    "discrim_hidden_size  = 128\n",
    "lr                   = 3e-3\n",
    "num_steps            = 20\n",
    "mini_batch_size      = 5\n",
    "ppo_epochs           = 4\n",
    "threshold_reward     = -200\n",
    "\n",
    "\n",
    "model         = ActorCritic(num_inputs, num_outputs, a2c_hidden_size).to(device)\n",
    "discriminator = Discriminator(num_inputs + num_outputs, discrim_hidden_size).to(device)\n",
    "\n",
    "discrim_criterion = nn.BCELoss()\n",
    "\n",
    "optimizer  = optim.Adam(model.parameters(), lr=lr)\n",
    "optimizer_discrim = optim.Adam(discriminator.parameters(), lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_rewards = []\n",
    "max_frames = 100000\n",
    "frame_idx = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAE/CAYAAABLrsQiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsnXl4XFd58H/v7NpGiyV5kSzvjmM7iZ04TkIWkhBCQqBJ2RqgJEDbUAptaft9FAoUCuXrvlGWNpQUKKUhFEgChGyQBJzdsR3HcuxYXmUt1r7OPnO+P+69o9HskmYkzej8nmcea869d+6Z6zv3Pe8uSik0Go1Gs3SxLfQENBqNRrOwaEGg0Wg0SxwtCDQajWaJowWBRqPRLHG0INBoNJoljhYEGo1Gs8TRgmCGiMh5InJARMZF5A8Wej6a4iIiT4rIby/0PDSaYqIFwcz5OPCEUqpGKfWlhZ5MMiJiF5G/FJFuU1jtF5E6c9u/ichEwisoIuMJxzaIyI9EZFJETovIe5I++z3m+KSI3C8iDfkeu1QRkbtF5KiIxETk/Unb3i8i0aT/k2sTtq8VkSdExCciR0Tkhizn+XsROWb+nx8RkTuStl8vIvtEZExETojIXQnbrhORV0RkREQGzf/Hlnw+W0QaReRp87gREXlWRK5M2J7rnvuoiOw1x7+Z5fv9uYio5GsgIjeY32tSRM6KyLvM8c0i8oCI9IvIkIg8IiLnJRznFpF/Mn8nwyLyVRFxJmyfSHpFReRfzW0uEflfETllzunapDm5ze99zjz3jxOv56JEKaVfM3gBjwO/nWW7fYHn95fAL4A1gADbAU+Gfb8J3JPw/n+A7wHVwFXAKLDN3LYNGAeuMbd/F7g3n2NnOH8BbAtw3RwZxp/M9v+dx+d+BHgDsBd4f9K29wN7shz7LPCPQAXwdmAEaMqw718AWzAWd5cBw8DrzG1O8//jQ+b1vRSYAC4yty8HVpl/u4G/BR7M87M9wHnmNgFuA4ayXM/ke+5t5jFfA76Z4ZgNwCtAN3BDwvhWoA+4GXAAy4AN5rbdwG8BDeb3/wJwJOHYzwK/Mrc3Ac8Bf5Hh/NXm9brGfO8CPmbe5z3AtUn7fxx42byuHuDbwA/n+56e0X260BMopRfGAzYKBMwbY7N5Y38NeAiYBG4AbgH2A2NAJ/C5hM9YCyjgA+a2YeB3zR/nQfPH/uWk834QeNXc9xFgTYb51Zvz2pDHd6nCeLC/PuF9CNicsM9/AX9t/v3/gO8mbNtg7l+T69g85vIk8EXgacAPbARqgW+YP7QuDAFnN/c/DVxi/v1e83paAuu3gPvNv3djPExHzM/5MuBKOK/CeFAfA06aY28EjmA8OL8MPMUcBEHCufYwA0Fg3ltBoCZh7FfA7+Z5vgeBPzH/Xm5+18qE7S8C705znBv4K+BwPp+dNG4D3mqeqznXPZe07S/JLAgeBt4MnGK6IPgu8IU8r0eDOa9l5vu9wDsTtr8H6Mxw7J3ACUDSbDtLqiD4GvC3Ce9vAY7O9R4q5kubhmaAUup6jB/jR5VS1Uqp18xN78F4kNVg/OAngTuAOoyb4MMiclvSx10GbAJ+A/hn4FMYQmQb8C4ReT2AiNwK/BnGyqnJPP//ZJjiBUAEeIeI9IrIayLykQz7vh3oB35pvt8MRBK+Exirmm3m39vM99a1OI758M/j2Hx4H3AXxjU8jSFgIxhCYSdwI2DZ6p8CrjX/fj3Gj/SahPdPmX9HgT8CGoErMFbmv5d03tsw/i+2ikgj8EPg0+Yxx4FEM0ebaf5om8H3ysVOERkw/68+IyIOc3wbcEIpNZ6wb17XVEQqMBYW7QBKqXMY98wHxDAdXoGhMe5JOKZNREYwBPH/wdAKcn52wvhBjAXSg8B/KKX60hyefM/lRETeCQSVUg+l2Xy5uc8rItIjIt9JNFcmcQ3Qq5QaTPz4pL9bRaQ2zbF3At9W5lM9D74BXCkiq0SkEmOx8rM8j10QtCAoDA8opZ5WSsWUUgGl1JNKqVfM9wcxfoSvTzrmC+a+j2IIjv9RSvUppbowHvY7zf1+F/grpdSrSqkIxsp8h4isSTOPVoyV9GZgHfAO4HMi8sY0+ybf3NUYGkwioxgPZmv7aIbtuY7Nh28qpdrN79iAsQL8mFJq0nyo/BNwu7nvU0xdz6sxVrDW+7ggUEq9pJR6TikVUUqdAv6d1P+Hv1JKDSml/OY525VS/6uUCmMI6F5rR6XUGaVUnVLqzAy+VzZ+iWG6a8Z4SL4b+L/mtmzXOxf/hiE0HkkY+x/gzzG0jF8Bn1JKdVobre+GIQA/jaEV5fvZKKUuBLwYi6I9aY6DGT5QRaQG437/wwy7tGIsIN6OsaiqAP41zee0Al8B/jhh+GHgD0WkSURWAFbgR2XSsWsw7plv5TNnk2MY2n4Xxu/ifODzMzh+3tGCoDB0Jr4RkctMJ1+/iIxiPMwbk445l/C3P837avPvNcC/mCvREQz7qwDpnE9+89/PK6X8phC6F+MBlzi/NowV9bcThicwfsiJeDFU+Vzbcx2bD4nXcA2GXbcn4Xv/O8YDE4wH/dUishKwA/dhrMDWYgjCAxB3GP7E1I7GMB4qyf8PieddlfjefGB1kidJzsWcWoNS6oRS6qS5YHgF42HxDnPzrK6piPwdhnB5l/XAFZEtGPfBHRj27W3Ax0XkljRzGsJ46D2QoJ1k/OykYwNKqf8BPiEiFyUdm+6ey8XngP8yhXg6/MB/KqVeU0pNYPz/Jt/rTcCjwFfNuVl8EcN8ewB4BrgfCDP9dwiGoNmjlDo5g3l/BcPEtgzDHPZDtEawJEj+UXwXQ0VerZSqxVhFScpR+dEJfMhciVqvCqXUM2n2PZhmPulWX+8DnlZKnUgYew1wiMimhLGLmDIBtJvvARCR9Rg3+2t5HJsPifPsxFi5NiZ8Z69SahuAUqoD8AG/D/xSKTWGsXK/C+NHGzM/52sYK9tNSikvhokt+f8h8bw9wOqE7yiJ73N+AcNcaL1mozWohPm1A+vNVbFF1msqIn+B4Ti90bwmFtuB15RSj5hC5yjwU3PfdDgwhG5cEGX57HQ4gfVJY+nuuVy8AfgDU5D3Yvxf3Ccif2puP0iWe11E6jGEwINKqS8mbjMXSh9VSrUopdYDg8BLCfeOxR3MTBsA2IGh4Q4ppYIYWspu0/S4KNGCoDjUAENKqYCI7MZQl2fLvwGfFJFtACJSa9pNUzDt9r8CPmWGsJ2PYU75SdKud2DY4BOPncRYuXxeRKrECAG8FcPpC/DfwFtF5GoRqcJYvf5QKTWex7EzQinVg/ED/gcR8YqITUQ2WH4Tk6eAjzLlD3gy6T0Y/w9jwIS5Kv5wjlP/FNgmIm8zV8N/AKyYzXewMEMNPRgPeKeIeETEZm67WUSWm39vAT4DPABg+lsOAJ81j/l14ELgBxnO80mM++yGJDs4GCvfTWKEkIqIbADegrlwML/veeZ1bsKIVNpvagdZP1tELheRq8zvWWE+pJcDzyfNIeWeM493mNfHDtjN72ppIm/AEGI7zFc3RuTTV8zt/4nh91hv2uI/gXmvi4gXw3z1tFLqE2nO22La8EVELse49p9N2ud1GJr399Mc7zbnDeAy520J8ReBO8zfqhPDL9WtlBpI/pxFw0y9y0v9RVI4IcbN/ZdJ+7wDw+E5jnFjfhn4jrltLcbKxZGw/7TIA+A7wKcT3r8PI3zOikK6J8v8WjDsnxMYTtQPJW2/AsMnUZPm2AYMFXkSOAO8J2n7e8zxSYwHVkM+x2LY8SfyvabmWC3Giv4shm18P3B7wvYPmddxjfn+Leb7yxL2uQZDI5jAEJCfJyFKx9x/Y9J5b8LQcFKihoA287PaZni/qKTXtea2v8cwRUya/1efB5wJx641j/cDR5keMfNeDH9G4ncJmvOzXn+WsP1dwCHznjwL/A1mmC6GZnXSnEcvhhlpTT6fjWE/f9n83CHzel0zg3vuc2muz+cyXMtTidfAHPsLDAd0P8bCo94cv9P8rMmkebcl3BunMDTLo8B705zv3zFMU5nmkjzvtea2ZRgLpz6MiLU9wO5iP5vm8hJz4hqNRqNZomjTkEaj0SxxtCDQaDSaJY4WBBqNRrPE0YJAo9FoljhaEGg0Gs0Sx5F7l8VNY2OjWrt27UJPQ6PRaBYdL7300oBSqinXfiUvCNauXcvevXsXehoajUaz6BCR0/nsp01DGo1Gs8TRgkCj0WiWOFoQaDQazRJHCwKNRqNZ4mhBoNFoNEscLQg0Go1miaMFgUaj0SxxtCDQaDSaJY4WBBqNRrPE0YJAo9FoFinPnxjk4UO9RGPFbSBW8iUmNBqNplz52lPHea13nBu3Li/qebRGoNFoNIuQ/vEgvzo2wK07W7DZpKjn0oJAo9FoFiE/OdhNNKb49Z0tRT+XFgQajUazCLn/QDdbV3rZvLym6OfSgkCj0WgWGSf6J3i5c2RetAEooiAQkb8TkSMiclBEfiQidQnbPikiHSJyVETelDB+kznWISKfKNbcNBqNZjFz/4FuRODXdqyal/MVUyN4DNiulLoQeA34JICIbAVuB7YBNwFfFRG7iNiBrwA3A1uBd5v7ajQazZJBKcX9+7t43YZlLPd65uWcRRMESqlHlVIR8+1zQKv5963AvUqpoFLqJNAB7DZfHUqpE0qpEHCvua9Go9EsGfadGeHMkI/bdsyPWQjmz0fwQeBn5t8tQGfCtrPmWKZxjUajWTI8cKALt8PGTdtXzNs555RQJiKPA+lm+yml1APmPp8CIsB/z+VcSee9C7gLoK2trVAfq9FoypiJYASHTfA47Qs9lYyEozF+/HI3b9y6nBqPc97OOydBoJS6Idt2EXk/8BbgDUopK0e6C1idsFurOUaW8eTz3g3cDbBr167i5l5rNJqy4L1ff471TdX802/sWOipZOSXr/Uz7AvPW7SQRTGjhm4CPg78mlLKl7DpQeB2EXGLyDpgE/AC8CKwSUTWiYgLw6H8YLHmp9Foyodnjw/S0TeRcfvgRJCXz47yq2MDTK1JFx8/2t9FfaWTazY3zet5i+kj+DJQAzwmIgdE5N8AlFLtwH3AYeBh4CNKqajpWP4o8AjwKnCfua9Go9FkJBZT/O53XuIvfpz5cfHiqSEABiaCnB32z9fUZsRYIMxjh8/x1otW4bTPb4pX0YrOKaU2Ztn2ReCLacYfAh4q1pw0moXEH4ryf77/MmuWVfLxm7Ys9HTKhpODk4z6w7xwcohAOJrWB/D8yaH43y+dHmZ1Q+V8TjEv7n3hDMFIjHftWp175wKjM4s1eTMeCC9qtXoxMxmM8IFvvsBPX+nh56/2LfR05oRSin989CivnRtf6KkAsP/MCADBSIyXTg+n3eeFk0Nctq6BKpc94z4LSSgS4549p7hi/TK2t9TO+/m1INDkxenBSS75wuPs6RhY6KmUHGOBMHfc8wIvnhpmy4oazg77Slqgdg75+dIvOvjOc6cXeioA7D8zTJXLjsMmae/PUX+Ywz1jXLFhGTva6th3ZvEJgp8c7KZ3LMBdr1+/IOfXgkCTF48dPkcoGqNzaHHaVxcrI74Qv/kfz/Ny5whffvdO3nFJK5OhKCO+8EJPbda0d48CLJoH6oHOEXa21XNxWz17jqUKgpdOD6EUXLZuGZe01fNqzxiTwUiaT1oYlFLc/csTbGqu5tp5dhJbaEGgyYsnj/YD4Astnh/QYmciGOHdX3+eIz3j/NtvXsLNF6yktd6wTS9Wh2U+HDIFwas94/hD0QWdiy8U4UjvODvb6rhqUyOHukcZngxN2+f5E0M47cLOtjp2rqknpuDlsyNpP++j393Hnz9waD6mHmdPxwBHesf5nWvWI1LcvgOZ0IJAk5PJYITnTw4CxsNNkx9Pdwzwas8Y//gbF3GD2WGqtb4CgLPDvmyHLmrau8cQgWhMcTDDA3W+eOXsKNGYYmdbHVdubEQpeOb44LR9nj85xEWtdXicdi5eXQ/AvjR+gt7RAD99pYefHeqdV9Pd3b88QXONm1vnqcBcOrQg0OTk6Y4BwlHjh+Fb4BVgKWGtTHe21cfHVpeDRtA1Fjdh7O9cWEFgnf+i1jouaq2lxuNgT0d/fPtkMMIrXaNctr4BgNpKJ5uaq9l3JnXePznYjVJGZ7Du0cC8zP9w9xi/OjbA+69ci9uxcBnPumexJidPHO2n2u3AbhOtEcyAYdMPUF85VSrAW+Ggxu2gayR/QRAIR/nKEx28a9fqBQ977BsLMDAR5OpNTZwcmGR/Bj+BUoo/+t4BescC1FW4qK9yUlvh4vyVNdxawGJqB86MsGZZJcuq3QBcsX5ZPGlMRNh3ZphoTLF73bL4MRe31fPI4V5iMTWtBeSDL3fj9TgYC0Q4cGaElrqKgs0zE1//1QkqXXbeu3tN0c+VDa0RaLKilOLJo31ctbGR2gonPi0I8mbYF8LtsFGRENcuIrTUV8zINPT4q+f411908JvfeJ6+8flZqWaivXsMgO0ttexsq2ffmZG0ZpRDXWPcf6CbwYkQx/snePzVPr6x5wR/eO+BgjlqlVLsOzPMztXxVidcvamRs8N+Tg8a1/eFk0PYbcIla6a0skvW1DPiC3NiYDI+dnJgkoNnR/nQ6zfgctg40FlYR/hkMMIf33eAL/zkMD/cd5ajveN0Dvn48cvd3H5pG7WV81dXKB1aI9Bk5ei5cXpGA3zshiZOD/mYCGrTUL4MT4aor3SlOABb6ytnJAie7hig0mWnfzzInfe8yL13XU5txcI8OA51GY7i81fWcHFbHT/a30XXiD/uBLd47HAvNoHvfegKGqpcANz3Yicf/8FBhn0hqtzpHz2RaIzesUDK56WjZzRA33hwmuntyo2NgOGAXdtYxfMnhti+ykt1wvkuXmMIjn1nhtnYXA3Ag2YjmLdd3MLPXz3HgQKbvH52qJcf7uvCZbcRisYAEAGbCB+4cm1BzzUbtEagycoTRwx767XnNVPlsuuooRkw7AtRl2al11pfwdlhf94OyT0dA1y5sZF/f98ldPSN89vfenHBonXau8dYu6ySGo8z/gBOZ29/9PA5dq1piAsBIH4tsoXO/nB/F9f/w1MMTgRzzsV6WO9I0AjWNVbRUlfBnmMDBMJRDnSOsHtdw7Tj1jdWU1vhjDuMlVI88HIXu9c2sLK2gh2r63mla5Sw+cAuBD852E1rfQXtn38Tj3zsGv7xXRfxwSvX8blf27bg5j7QgkCTgyeO9rF1pZflXg9Vbseiir9e7Az7wtMehBat9RVMBCOM+nPnEpwZ9NE55OeqjY1cvamJf/6Nnew9PcxHvruvoA+qfGnvGWWbmfm6ZUUNHqctxU/QOeTjSO84bzQjpSzqzWsx7Jse3pnI2SEfoUiMQ6YJKhv7zwzjctg4f6U3PiYiXLWxkWeOD7Dv9DChaIzLEvwDADabEUpq5UG0d49xon8y3hZyR1sdgXCMo72FyZwe8YXYc2yAWy5cidNu47wVNbzt4lY+85atvO/yhfUNWGhBoMnIqD/MS6eHuW6LESFS5bYzqaOG8mbYZ5iGkplJLsHTx40EqSs3Gg+zWy5cyRdvu4BfHOnjM/fPb7z7qC9M55CfbauMB6/DbuPC1roUjeDRw+cAUgRBnWnOGs6iEQyZQsIyQWVj/5kRLmipxeWY/hi7alMjY4EI/7HnJCJw6dqGlGMvaavntXMTjPrDPPhyNw6b8ObtKwHiPodMEVE/OdjN/3voVZ45PpCXMH6kvZdITPGWCxYuPDQXWhBoMrLn2ADRmOK685oBqHJpjWAmDE9mNg1BfrkEezoGWO51s6GpOj72nsva+O2r1vG9vZ109M1fvZ/2HuPhvG3VVC2ci9vqOdw9SiA8tUB47HAvm5dXs7axatrxdaZQHM2iEQxPGkLicA6NIByN8UrX6DRHscXrNhhC8xdH+tiywpvWEWs5j/edGebHL3dzzeamuMbSWl/BsioXB9KYvMLRGJ99oJ27f3mC93z9eS7+/GN85L/38cCBLmKx9Ka+nxzsoa2hku0t3rTbFwNaEGgy8sTRPmornHEbrDYN5U8sphj1ZzYNQW6NIBZTPHt8kCs3NqY4nH/vuo14HHa++sTxwk06B+1dxsPZ0ggAdrbVEY6qeNmJEV+IF08Np2gDMOUjyKYRWGYj6/MycaRnnGAkxo62VEGwrNodn+Nl61K1AYCLVtdhE/j3p47TMxqYlswlYpiO0kUO7Tk2wOBkiH+5fQf//r5LePMFK3nh1BB/eO8B/vOZUyn7D02GeOb4IG+5cOWCZQ3ngxYEmrTEYoonj/ZzzeYmHGZtdMs0VMoF0+aLsUCYmJpaBSdSW+Gk2u3IKQhe7R1jaDLElRsaU7Y1VLl472VtPPByN2cG5ydLub17lBVeD41mzD4YggCmKoD+4kgf0ZjijVtTO9g67TZq3I6sPoIhMwnv1KCP8UBmgbHffEgnRgwlctUm45olO4otqtwOtqzw8tyJISqcdm44f7rg2rG6juP9kyl+nPsPdFFX6eTm7St507YV/M07LuT5T76Bqzc18qWfH2Mk6bs9fKiXaExxy4UrM36XxYAWBJq0tHePMTAR5LrzpopgVbkdRGOKYGT+nZSlhvVAa6hKNUuIiBk5lP0B/nSH5R9IFQQAv3PNeuwifO2p2WsFZ4d9eRePa+8eSzFvNNd4aK2viH/Go+3naK5xc2GGUsp1Vc6sUUMjvjBNNYagyWYe2n9mhOYaN6tqPWm3v/3iVq5YvyzjtYOpMNIbti5PCWfdYZaiSCyhMRGM8Eh7L7dcsHKaX8JmEz59y1bGA2H++fFj0z7np690s76xiq0rF69ZCLQg0GTgiaN9iDCtZV6Vy/ixaPNQbizzRzqNAKZCSLPxdMcgG5urWZHhYbfc6+Gdu1r5wUtn6Z1FSYRnOga45Ut7uOvbe3Pu6w9FOd4/wdZVqQ/4i9vq2X9mhEA4yi+P9XPD1uXTMnYTqa90ZdQIlFIM+UJcZT6827MIggOdI+xYXZfR3LJ5eQ3/kyPfwnIi33pRqhP3wtW1iExpOgCPtvcSCMfS9hM+b0UNt+9u4zvPneZ4v9Eys388yLPHB7llkZuFQAsCTQb2nxnmvOU108wA1qpJ1xvKjVVnKF3UEFhJZZlzCYKRKC+cHOLKDcvSbrf43ddvIGqWMZ4J333+DHfc8wLjgTADEyFCObS8V3vHiKnp/gGLnW119IwG+MG+s/hCUW5M4x+wqKt0ZfQR+MNRQpEYm837LpMgGJ4McXJgMqNZKF/efMFK7n7fJbzh/OaUbV6Pkw1N1dMSy+4/YOQCJGYpJ/JHN2zG47TzVw+9CsDD7b3EFIveLARaEGgy0D0SSEl0qXIZpRLKqd5QNKZ4x9ee4eszfJD2jQX40/89OC1aJhFr1duQRSPIlkuw/8wI/nA0q2kDYHVDJbftaOG7L5zOKwkrGlN8/seH+bMfvcJVmxr5xM1Gy8yBHMe2m+Gc6bpnXWw+kL/082NUux1ckUV41VU4U+zoFonmtO0t3owOY+vhvDONo3gmOO02bty2IuNqfcfqOg50GiU0+sYD7DnWz207WjLu31Tj5iPXbeTxV/t4umOAn7zczcbmas5bXjOnec4HWhBoUlBK0TXiTym6NaURlI8gePhQL3tPD8ebm+fLsycG+d7ezoyrVssOXpfGRwC5I4ee6RjAJnB5Do0A4Peu20AwEuOep09m3S8SjXHXt/dyz9Mn+cCVa/mPO3axvtEIS+0bzyEIuseoq3Smtcmfv9KL22Hj3FiQ129uylpFs77SmdIvwGIkXqTPxbZVXo71TaQVtE8c7cPjtHFR69wEQS52rK5jaDJE55CfH7/cQ0zBbTuz5wJ84Mq1tNZX8On7D/HCqSFuuWDxm4VACwJNGsYCESaCkfjDyqLKbWkE5WEaUkrx1Sc7AOjPYzWdiFXioW8svW1+yBfCYRNqMtTUmUoqS+8w3tMxwIWtdXg9uWsKbWiq5s3bV/LtZ05nzVZ+9sQgPz/Sxydu3sJn37oNh91Gs9ed9XtYtHePsW2VN+1DzeWwcYGpKaQLG02krtLFWCBCNE3MvaUR1Fe52LaqlmhMpfRFjsUUDx/q5drNzVS4ilu2eUc8sWyYBw50sb3Fy8bm7Kt7j9POJ27ewsmBSZSCt5SAWQi0INCkoctcpa7KpBGUiWnol8cGaO8eo8bjoG9sZoLAWqmey/AAHfGFqEtTcM4im0YwFgjz8tnRuNM0H37vug2MByP84KWzGfd56JVeqlx23v+6tfExK0InmyAMR41yC9vTOIotLl3XgMthiycfZsIqyZ1OYFnmtPpKV/xcyRrX/s5h+saD3LQ9NTy10FglNH64r4uDZ0e5Lc/y2bdcsJLdaxvY3uJlUwmYhUBXH9WkoduslZ9iGjKjhsrFR/CVJzpYWevhpu0r+O/nz8Rr2OeDP2w4VzOZVIYmQ9P6ECSTLZfg+RNDRGMqp38gkW2ratne4uX+A1188Kp1Kdsj0RiPtvfyhvOX40koi20FA2QThMfOTRCKxtiaxlFs8dHrNvL2i1tyllNOrDeUnGw35WB30lDlosbjSCk18fChXpx24fo0Dt5C47DbuLCljqde68cm8NY00UXpEBG+9cHdRGKlE2atNQJNClbTlEwaQTmEj750eogXTg7x21evp6WuglAkxpg//+/lj2sE6R+gw75w/KGXjqlcglRB8HTHAB6nLR7nni+37Wjh4NlROvomUra9cHKIwckQb75g+kraabfRUOXKqhFYTtttWTSCKrcjp9kEpsJp0zmMh3xhRAwhKSJsXemdphEopfjZoV6u2tiYl8msEFiZy6/b0Mhyb/ow3nRUuOzUzNMcC4EWBJoUukb8uBw2GqunP8gqTZtsORSe++oTx6mvdPLu3asTzCP5x+JbpqFMjWJGfNk1AiBtUlk0pnikvZfXbWiccevCX7toFTaBBw50pWx76FAPFU47r9+cupJurnFn1QiO9I7jcdpYl1Q7aDZY18SqKZTIiC+E1+OMZ7JvW1XLkd6xuD+hvXuMs8N+bt4+f3Z3q5bRQvYTng+0INCkYEUMJZtJ3A4bDpuUvEbwas8YPz/SxweuXEely0HeZN7nAAAgAElEQVRzjbHSm4mfwHIWZ/IRDE2GM+YQWLTWV9KVlEvwdMcAPaMB3n5xa95zsWj2erhqUxM/2j+9AFo0pnj40Dmu35LewdpU46Y/S+ezrmGj8Yw9Q5LYTKiryFyKemhyurloe4uXQDjGCTNB62eHerDbJKdDupDcsHU5f//Oi7gtTRJZOVF0QSAifyIiSkQazfciIl8SkQ4ROSgiFyfse6eIHDNfdxZ7bpr0dA2nho6CYc6odNlLPqHsa08ep8pl584r1gL5OUyTCWQxDSmlDI0gi2kIDI1gPBiZZpK6b28ndZVObtg6Oxv4r+9cxdlhPy8llI3Ye2qIgYkgN1+Q3sFqCILM37171J9iJpwtVjhtujITI77wtGqtlinqUPdo3Cx0+fqGnNe1kDjtNt5xSStOe3mvmYv67URkNXAjcCZh+GZgk/m6C/iauW8D8FngMmA38FkRmVvqoGZWdI/4WVWX3h5a7XaUtLP49OAkPznYzXsvXxN3bE6FUM5AIzAFwag/nBLrPhGMEImpvExDAJ2meWjUF+bRw+e4bUfLjM1CFjduXUGF086P9k+Zh352qBd3loie5hoP/RPBjFnO3SN+WjLcDzOlxu3AYRNG/Bk0ggQtakNTFW6HjfauMTr6JjjRP8lN24ofLbQUKbaY+yfg40DiHXYr8G1l8BxQJyIrgTcBjymlhpRSw8BjwE1Fnp8miWAkSt94kJa69O3zKt2Okk4ou39/Nwr4rYTImhq3A7fDNiuNAEhZTVv271ymIesaWw7jB1/uIhSJ8Y5LZm4WsqhyO3jTtuX89GAPwUiUWEzxs0M9XHteU8Y+wc01bsJRlbb0QyAcZWAixKrawmgEIkJdpTPtuZK1KIfdxpYVNbR3j/GzQ72IwJu0ICgKRRMEInIr0KWUejlpUwvQmfD+rDmWaVwzj1jFyzJpBFVuR0knlD17YiDeetNCRGj2unMmVSXiTxAEyX6CxHj4bCQ3qLlv71m2rvSmLeMwE27b2cKoP8yTR/vZ3znMubEgb74gs4M1bhpLYx7qMe+HlQUyDYEROZQ+aijVwb6tpZZD3aM89EoPl7TV0zyDyB1N/sxJEIjI4yJyKM3rVuDPgD8vzDRTznuXiOwVkb39/f3FOMWSxUoma6lP/8OvctlLNqEsEI6y78wIV6xPLdvQVO2ekUbgD0XjMfjJfgKr3WJ9hvISFnWVTqpcds4O+3m1Z4xXukZ5567ZawMWV21spLHazY/2dfHTg7247Dau35LZ59BsCoJ0EVA98VDiwj2AjTIT0zUCfyhKIBxLsf9vW+VlPBDhSO/4vCSRLVXmlFCmlLoh3biIXACsA142I09agX0ishvoAlYn7N5qjnUB1yaNP5nhvHcDdwPs2rVLd0kpIF0ZksksqtwOhibnpxFKodl3ZphQJJa2KFpzjSdePjgfAuEYa5ZVMjARTHmAjuSpERi5BEYV0u/vPYvLbss7ezUbDruNX7toFd957jS1lU6u2dyYNabdWmWn0why3Q+zobbClRI2m0mLSsxd0IKgeBTFNKSUekUp1ayUWquUWoth5rlYKdULPAjcYUYPXQ6MKqV6gEeAG0Wk3nQS32iOaeaRrhE/ImSsgV9VwlFDzx0fxCZGOYRkmmrcOQuvJRIIR1lZ68FplxSNIF8fARjmoZMDE9x/oIsbtjYXLCLm13e2EIrG6B/PbhaCKdNQuu/fPWIIuUz3w2yor0xtTjOUoWz3lhU12G3Cha218fpMmsKzECUmHgLeDHQAPuADAEqpIRH5AvCiud/nlVIzKwmpmTNdw36aqt0Zo1ZKuW/xsycGuaClNm1WanONm1F/mGAkmlfEjj8cpcJpp7nGk+JbGPaFEAFvlqYoFq31Ffz8SB8A77xkdY6982d7i5cNTVWcGfLxhvOzx91Xux1Uuuxpo6a6R/w01WS+H2ZDfVVqc5qpyqPTr5nHaecj123kota5+U002ZkXQWBqBdbfCvhIhv3uAe6Zjzlp0tM96s/oHwBTEJRg1JA/FOVA50jaOjww3WGaz8rTH45S4bKz3Ovm3HiqIKircOaVgGWda7nXzdWb8q8tlAsR4c/fuo0zQ76sXbosmmvS+0i6R/0Z20HOlrpKJ8FIDH8oGk9ws/wqyfWHAP74jZsLen5NKrronGYaXcP+rFErVS4HgXCMSDQWLwVQCuw9PUQ4qtI6imEqlyBvQRCa0giSfQvDvtxZxRZW5NDbLm4t+PV8fUKb0Vw01aSPmuoe8bO5wBU0rWsz4g9R4TK+f9yvMo/JYpopSueXrCk6sZiiezSQ1TFo9STwZejMtVh59vggdpvE+9QmEy8zkYefIBZTBCMxPE5TI0g2DU3mziq22LW2gWs2N/Gbl6/Ja/9iYSWVJaKUonskULCsYot09YYsH0FdHtqLpvBoQaCJMzAZJBSJ5TQNQelVIH32xCAXttZmTKrKFkufTNDs71vhstPs9TAWiMRrD4GlEeT3QGuqcfPtD+4uaFTObGiqcdOf5CMY8YXxh6MFFwTpKpCO+MJ4PY6S0jLLCX3VNXGsCJFsWaTxCqQllFQ2EYxw8OxoRrMQwLIqFyL5aQRWMlmF0x5PTEsMIR2eDMUfdqVCU42b8eB0gTYVOlp4HwEwLbt4aAZalKbwaEGgiZMrmQyMCBMoLY3gxVNGo5dsTdUddhvLqlx5aQSWIPA4bQnJWFPHpWu6sthpTqMRxbOKC1RewsLyESRGDg37Qnn7VTSFRwsCTZzuDA1pEqk0u5SVUuTQcycGcdqFXWvS+wcsmmo8WcsxW1irZk+CRmD5CfyhKMFIbFoVzVKgKU12cT73w2ywrs1IiiAorWtWTmhBoInTNeKnxu3IGm44pRGUjmnoueOD7Fhdl7PZea5yzBaBaaah6WUm4mGQJba6Tecs7zYbFC0rsHbjdtipdNmnmYaGJ7N3dNMUFy0INHG6RnLXna+0ooZKRCMYC4R5pSu7f8CiOc/s4rggcNmprXDictjioZdW391S8xEkhs9adI0YOQS2AjSkSaa+0jUtu1ibhhYWLQg0cbqGsyeTwZRGUCo9CV48OURMweVZ/AMWTTVuBiaC07p7pSPRWSwi0wTIVM2c0jJzNFS6sNskxTRUaLOQRV2lM24aCoSj+ELRkvOrlBNaEGjidGVpSGNhRQ35SsQ09OzxQVwOGxe35e5xZNXlH/Gn1spPJNFHALDc64n7CCxzR6k91Gw2obHaleIsLrSj2KK+cqrMxFR5idK6ZuWEFgQawFjhj/rDGRvSWFjO4lLQCGIxxZ6OAS5uq4s/tLORby7BVNSQJQimksqsVW6pmYbA8BNYmk04GuPcWKDgoaMWtQmF56YKzpWWFlVOaEGgARIjRLL/8O02ocJpX/Q+AqUUX/jpYY70jnNrnqWdpxym2SOHEn0E1nFWwbZ4hmwJPtSMMhPG9zg3FiCmCh8xZFFf6UzQCHR5iYVGCwINMJU81JrDRwCl0aXsK0908J9Pn+KDV67j9kvzq+qZt0YQmvIRgGEaGg9G8IUijPjC1HgcJdnsPLHwXDy5sGiCwMWoP0wspqYa+ZSgFlUulN7dqikKVjJZPj/8Krd9USeUfee50/z9o6/xtp0tfPqW8zGbI+UkXXJYOgJWiQmnpRGYx40FSzr6panGzeBEkGhM5a0hzpa6ShcxZUR1WZFWuTq6aYqHFgQawDANOWwSN49ko8q1eBvY//RgD5954BDXb2nmb95x4YxCH6vMuvz5agRuh/HzSUwqK+VSCc01bmIKBieDdI8agqB4zuKpMhOWg72uojSvWzmgBYEGMExDK+s8edXQr3LbF6Wz+EDnCB/73n52rannK++5eFbmmXxyCQLhKG6HLS5k4kll40FGZlBwbrHRZPlIxoJ0j/iNnsoZivTNlfqEwnNDkyFq3A5cDv04Wij0ldcAZsx4nqu/KrdjUbarfOpoP+Go4ut37MqZRZyJ5jzKTFhNaeLHWIXnLI2ghE1DYPhIukcCed8Ps2GqzESYEV+IOm0WWlC0INAA+SWTWVS5HItSIxj1h6ly2ecUuplP72KrKY2F1+PA7bDRNx5kpIR9BM3TBEHxkslgKrx22BdiyBcuuZIc5YYWBBrC0Ri9Y9kb0iRS5bYvyoSyUX84r7aM2cin3lAgEpsmCESE5V4PnUM+JkPREjYNTRWe684juXAuJPoIRnyl61cpF7Qg0MRjxvMXBIuzgf2oP5xXw/hsNNW4GQ9E4rkC6fCHoikJasu9bo72jgOlGw/vcdrxehycGJhkLBApqkbg9TixyZSPoFS1qHJBCwJNPCt2eZ5NyqtcRgN7pbLX5JlvxgqgEaSry59MIMlHAIaf4NTgJFDa8fDNXg8vd44AxcshAKOkRW2FkVQ2MoMez5rioAWBJv7Qa6p257V/ldtBTEEgHCvmtGZMoUxDkD272B+O4nFO/+lYoZdQ2qUSmmvcHO83BFqxyktY1Fe66BsLMhGMlPQ1Kwe0INDQP2Ek9Fir4VxYDewXW3OaQpiGrDyKbBpBsrMYpnIJoHRNQzAlCKG4GgEYkUNxLaqEr1k5oAWBhv7xICL5V8ysci3OdpVjgUJqBNlNQ+l8BBalbOawFgP2PJML50JdpYtTAz6gtK9ZOaAFgYaBiSANlS4ceSZgxTWCRRQ5FI7G8IWicxYEDVVGXf6cPoJkQZDw0CzFgnMW1sN/hTe/5MK5UFfpJBQ1zIu6vMTCUlRBICK/LyJHRKRdRP42YfyTItIhIkdF5E0J4zeZYx0i8olizk0zxcB4kMY8/QNAPNt0MZmGRs0eAnMVBHabsKzKFa/CmY7khDKY6vBV6bLnVfJ6sWJpRMUMHbVI1AJKrX9DuVGc/HFARK4DbgUuUkoFRaTZHN8K3A5sA1YBj4vIZvOwrwBvBM4CL4rIg0qpw8Wao8agfyJIY03+P8TKRWgaKpQgAOOhblXhTIc/jUZgZReXuomjOS4IiusfgOlO9VK/bqVOMTWCDwN/rZQKAiil+szxW4F7lVJBpdRJoAPYbb46lFInlFIh4F5zX02RGZgI5h0xBIuzgX0hBUFTtTtj1FAspgiEY7iTBEGN20GF017yJg5LIyhWsblEEjPAS9mcVg4UUxBsBq4WkedF5CkRudQcbwE6E/Y7a45lGtcUEaUUA+OhGZmGrHaVi9E0NNeoIbDqDaXXCIJJJagtjOxid8mvbFvqK6itcHJRa23Rz2VdqyqXHbejdM1p5cCcTEMi8jiwIs2mT5mf3QBcDlwK3Cci6+dyvoTz3gXcBdDW1laIj1yyTIai+MPRaWGDuZjSCBaPIBgrpEZQ42ZgIkQ0plIcpvHuZM7UNdTHbthMbYmvbCtdDvZ95o0U2U8MTJmGdOjowjMnQaCUuiHTNhH5MPBDZaSfviAiMaAR6AISW0a1mmNkGU8+793A3QC7du1aXOmtJcaAufKdkUZgRg0tpgqkhfYRRGOKoclQioD0J7WpTOS2neWhwBY7WsjCEpqlrkWVA8U0Dd0PXAdgOoNdwADwIHC7iLhFZB2wCXgBeBHYJCLrRMSF4VB+sIjz00DcKdo4A43A7bDjtMuiqkA66iucILAiWKxeuokkN67XzB5LAGiNYOEpWtQQcA9wj4gcAkLAnaZ20C4i9wGHgQjwEaVUFEBEPgo8AtiBe5RS7UWcn4YpjWAmzmIwTAi+xSQI/GEqnPaCNDfxegxhMhYIp2yzupNpQTB34oKgxM1p5UDRBIEZ+fObGbZ9EfhimvGHgIeKNSdNKgNxjWBmq7LqRdbAvhB1hiwsh/OYP1XQTfkItCCYKxUuOzVuR96lTTTFo5gagaYE6B8PYhNYVjVTjcC+qPoWF1QQeBzxz0wmm49AM3O+/Vu7Wd1QudDTWPJoQbDE6Z8IxcsqzIQq9+LqUlZIQWB9TjrTkFVxVWsEhWFnW/1CT0GDrjW05BmYmFl5CYsqt31RhY8WovKoRY3lI8iiEWgfgaac0IJgidM/HpxRDoFFlWtxNbAvRFMaC5fDRoXTntY0FAhp05Cm/NCCYIkze42gfE1DYJiH0jmL4xpBAaKTNJrFgr6blzBKqdlrBG57XhpBLKYIRoqrOYSjMSYLUII6EW+FI334qHYWa8oQLQiWMBPBCMFIjMbqmSf0VLny0wi+/EQHb/3XPbOZXt5MlZcoXOyD1+NMbxqKawRaEGjKBy0IljADZovK2ZqGQpEY4Wj2vsWnBid57dxEUUNN4+UlCpiYVFvhzKgRuB02bPNUhkGjmQ+0IFjCxJvWz8I0ZFUg9eVIKrMii84O+2d8jnwpZJ0hC28GH0EglNqURqMpdbQgWMLEs4pnoRFU59mlzPIjnBn0zfgc+TIWMOZglYYoBF6PI2NCmc4h0JQbWhAsYeakEeRZitryI5wZKp4gKIZGUFvhZDwQJhabXtzWH47pHAJN2aEFwRJmYMIoLzGbMsDVVgP7HJFDlqDoHC4tQeCtcBJTqRpPIBzVgkBTdmhBsIQZmAjSUOWeVf35fPsWW+0sO4uoEYwVsDuZhWVmSjYPBcLRtE1pNJpSRt/RS5jZ5hBA/l3KrBV159DMnMXnxgJEckQkWYz6w7gdtoKu1L1mKGqyw9ivncWaMkQLgiVM/0RoVjkEkH/fYiuq6MyQD6MdRW6CkShv+Ien+PqvTua1/6ivsFnFkFCKOimEVDuLNeWIFgRLmIHx4Iwb0lhMaQSZfQShSIxQNEZjtRt/OBrPW8jFudEgE8EITxzty2v/QpeXgMymIX84ilsLAk2ZoQXBEkUpRf/E7E1D+UQNWUlk56+sAfJ3GPeOBQDYf2Y43hEsG8UQBPFS1Mk+gpDWCDTlhxYES5TxYIRQJDarHAKASmfuqCErdHTLClMQ5Okw7hk1/AnhqGLv6aGc+xdFI4ibhpKihiIxLQg0ZYcWBEuUueQQANhsQqUre08CK5nsvBVeIP+kst5RQyOwCTxzfDDn/sUQBDVuByJpTEPaWawpQ3SHsiWK1bR+thoBGPWGstUQsjSCZdUummrcMzINVbnsbFnp5dk8BMFYAZvSWNhsQrXbMc00pJTCr/MINGWI1giWKPGCczNsWp9IlcuetYG9pS1Uux20NVTmnV3cOxpgRa2HKzcs4+DZkbTF3yyiMcV4MFJwjQBSC88FI7pNpaY80YJgidI/bphfZhs1BKZGkMU0ZEUUVbrstDVU5p1L0DtmCILLNywjpuDFk5n9BGNFyCq28Hqc0zQCy3Ht0QllmjJD39FLlIGJEHabzKq8hEWungSJGsHq+gp6Rv2EIrmTxHpHA6zwVnBxWz0uhy2rn6AY5SUsvBWOaQllAbPBjtYINOWGFgRLlIGJIMuqXHOqq5+rS5nlP6h0OVjdUElMQfdIdq0gGlP0jQdZUevG47Sza039ggmCZNOQX/cr1pQpWhAsUfrHZ9erOJFKtyNr1JDlP7B8BJA7l2BgIkg0plhRWwHA6zYs49WeMYYn0yejFaMpjUVyl7J4v2KtEWjKDC0IligDE0EaZxk6alHjdqTE2SfiC0WwiWFTX20KglwO4x4zdHSF1wPAFRuWAfDcifRaQXFNQ9N9BFabSm0a0pQbRRMEIrJDRJ4TkQMisldEdpvjIiJfEpEOETkoIhcnHHOniBwzX3cWa24as+DcHDUCr1mzPxMTwQhVLgciwnKvB5fdllMQWDkEK2sNQXBhax2VLntG81BRBYHHyWQoGi9+5w+ZUUPaNKQpM4qpEfwt8BdKqR3An5vvAW4GNpmvu4CvAYhIA/BZ4DJgN/BZEakv4vyWLEopBiZCcwodBaOLVzASIxhJ7yfwBaNUmn0L7Dahpb6Cszkih3rNrOIVpiBw2m3sXtfAM8cH0u5fXB+BkWYzbmo9ft24XlOmFFMQKMBr/l0LdJt/3wp8Wxk8B9SJyErgTcBjSqkhpdQw8BhwUxHnt2QZC0QIRWNz1ghqzMJs4xnMQxOhCFXuqZzF1XnkEvSMBXDahYaEaKbXbVjG8f5Jzpk1iBIZ84dxFbgEtYWVpGYJm7hpyKUtqpryoph39MeAvxORTuDvgU+a4y1AZ8J+Z82xTOOaAjPX8hIWUzX705uHfMFIvEopQFtDRU5n8bnRAMu9nmnRTFesbwTS+wnGAoUvL2FhVSC1Ioe0s1hTrsxJEIjI4yJyKM3rVuDDwB8ppVYDfwR8oxATNs97l+l32Nvf31+oj10yzKVpfSLeHBrBZDAa71sAsLq+khFfOG1TeIue0UDcUWyxdZUXr8fBMx2pgmDUH8brKU6lFCsSycol0M5iTbkyJ0GglLpBKbU9zesB4E7gh+au38ew+wN0AasTPqbVHMs0nu68dyuldimldjU1Nc3lKyxJLEEwd40gffMWi4kUjcAMIc1iHjpnZhUnYrcJl69fxtNp/ATFKDhnkdyTQOcRaMqVYpqGuoHXm39fDxwz/34QuMOMHrocGFVK9QCPADeKSL3pJL7RHNMUmP4CFJwDqPGkb+do4QtF4r2NgXgI6dkM5iGlFD2jgXjEUCJXbFjG2WF/yrFFFQSW6SvZNKSdxZoyo5jVR38H+BcRcQABjAghgIeANwMdgA/4AIBSakhEvgC8aO73eaVU7mL0mhkzMBHEYRPq5vgAnTINZdIIoinOYsicSzDqDxOMxFjuTRUEl6838gmePzFE6yWV047Z2FQ9uy+Qg+TmNIFwDJfDNqdsbI1mMVI0QaCU2gNckmZcAR/JcMw9wD3FmpPGoH88yLLquZWXgNymIV8oQlWCGaW2wklthTOjIOiJ5xBUpGw7b3kNdZVOnjsxyNsvaY2PF6NfsUWF047DJtOihrR/QFOO6Di4JUghykuAUYbaJulNQ7GYwhearhEArG6oyFiF1EomW1GbOjebTdi9toHnTk45jGNFLEENICJGdnFgykegBYGmHNGCYAlyZsjH6vrK3DvmQESo8aTPLvaZ9vQq9/QHp1GOOr1GYPUqXpFGIwDDPNQ55KfLLFw3HoigFAVvSpNIbYUzLuj8Yd2dTFOeaEGwxIjFFJ3DftYsm7sgALNUc5rwUasYXapGUMnZYT+xmEo5pmc0gAg0Z4hmumx9AwDPm/kExcwqtvB6HFNRQ7o7maZM0YJgidE7FiAUidFWIEFQ43amTShL7EWQyOr6SkLRGOfGU7OEz40GaKx247Snvy3PX+GltsIZTyybF0GQYBoKhKO6KY2mLNF39RLjtNlAfk1DVUE+z9AI0gkCqzvZdEFg5RKka2TfM5Y+dNTCZhN2r2vgebNj2bwJAu0s1pQ5WhAsMU4PTgIUzjTkcabNLJ6Im4ZSfQSQPoS0d9SfNnQ0kcvWNXB60Ef3iL+ovQgsjJ4ECT4CLQg0ZYgWBEuM00M+nHZhVV16h+xMSa7Zb2F1J6tK0gha6iuodjvYd2Y45ZjeDMlkicTzCU4OzpNG4JgWNeTRzmJNGaIFwRLjzKCP1vpK7AVKiqrxpHcWT2RwFjvtNq7e1MgTR/oxUkoMfKEIY4FITo3g/JVG3aHnTwzNk7PYSSgSIxCOEgjHtEagKUu0IFhinB6ajJtnCoHX42QiGCGaFAVk9TJONg0BXLelmd6xAId7xuJjyQ1pMmG3CbvXLeO5E4ZG4LRLUR/OidnF2jSkKVe0IFhCKKU4PegrmH8ApmL4J5K0gkzhowDXnmcUCnziSF98bCqZLLsgALh8fQOnBn28dm6c2gonIsUr+ZCYPa2jhjTlir6rlxAjvjDjgUhBNYJ44bmkyKF41FCaFXRzjYcLW2v5RYIgSO5VnA3LT7CnY6CoyWRAvMT1qNYINGWMFgRLiFNmxNDaZYUJHYXU5i0Wk6EIHqcNR4acgOvOa2Z/5whDkyEgMas4tyA4f6WXGo+DUCRWVP8ATJmG+seDKIV2FmvKEi0IlhBWyGZhTUPpS1FPmo3rM3H9lmaUgqdeM7SC3tEAXo8jJe8gHXaz7hAU11EMU6ahc2NG6W6tEWjKES0IyoyTA5OcGphMu81KJltdYGcxpDMNRdL6BywuaKmlsdrNL44YHeZ6xwJpq45mwjIPFV0QeCxBYGgsWhBoyhEtCMoIpRQf/OaL/MG9+9NuPz3oY4XXU9B6OZnaVSb3IkjGZhOuPa+Jp472EYnG6B0NsDwPs5DFvAkCU+OxNAJda0hTjmhBUEa8eGqYkwOTHOoajcfxJ3JmaLJgNYYsMjWwT+5FkI7rtzQzFoiw78yIoRHk4Si22LrKy/rGKs5f6Z35pGeA22HH47TRZ9ZG0oJAU45oQVBGfH9vJwAxBftOp2bunhr0saaAZiGYKio3U9MQwFWbGnHYhEfbexmYCOblKLaw24Rf/J9reffutplPeoZ4Pc4p05B2FmvKEC0IyoTJYISfvtLDLResxCaw99T0Lp++UIT+8SBrGwsXMQTgsNuoctlTTEOToWjaZLJEvB4nl65t4If7u1Aqv4ihhaC2whnPc9A+Ak05ogVBmfDQKz34QlHef+Vatq7y8uKp6RqBFTFUyBwCi3T1hnJFDVlcv6U5HkK6WAWBUYraEHRaEGjKES0IyoTvv3SWdY1V7FpTz6VrG9jfOUwoEotvj5efLrCPAIyV/WxMQ2CUm7DIJ5lsIbCSygAqXPonoyk/9F1dBpwamOSFk0O845JWRIRL1zYQCMdo7x6N73OmwH0IEqnxOKblESil8jINAWxoqoprKbnqDC0UidnLbofWCDTlhxYEZcD/vnQWm8DbLm4BYNfaegD2JpiHTg9NUlvhLErtfm+Fk/HglEYQjMSIxlReyWEiws3bV7CsylX0UNDZkjgv7SzWlCNaEBSYw91jfPg7LxGMROflfNGY4gf7znL1pqZ4QlZzjYe1yyp5IcFhXOhic4l4kzSCTG0qM/HHN27mZ394dVGLx80FK1cCtI9AU55oQVBgHm7v5WeHejnUNZZ75wLwdMcAPaMB3rmrddr4rrUN7D01FK/5bwiCwpuFAGqSfARTJajzE4GRXH0AABjUSURBVARuh53mReofgKlcCdB5BJryRAuCAnO8fwKAwwn2+WLy/ZfOUlvh5Ibzl08bv3RtPcO+MMf7JwlHY3SN+AueQ2DhrXAwHojEhU68KU2ZmFEs05DLYStYQx+NZjGR35JNkzfH+wxBMB8awagvzCPtvdx+6eqUleqlZlG2F08N4bQL0ZgqeFaxhdfjJBpT+EJGWYlsvQhKEcs0pM1CmnJlThqBiLxTRNpFJCYiu5K2fVJEOkTkqIi8KWH8JnOsQ0Q+kTC+TkSeN8e/JyKuucxtIYjGFCfMgm/tPcXXCL76VAehSIzfuHR1yrZ1jVUsq3Lx4qmhqdDRomkE0wvPTWbpTlaKWN9PN6XRlCtzvbMPAW8Dfpk4KCJbgduBbcBNwFdFxC4iduArwM3AVuDd5r4AfwP8k1JqIzAM/NYc5zbvdA75CEViNFa7Odo7Pi2Ov9B09I3zjV+d5J2XtLJtVW3KdhFh19p69p4a5nS8/HSxfATGyt/KLi43jcAyDWmNQFOuzEkQKKVeVUodTbPpVuBepVRQKXUS6AB2m68OpdQJpVQIuBe4VYxwkeuB/zWP/xZw21zmthBY/oG3XrSScFRxrG+8KOdRSvGZ+9updNn505u3ZNzv0rUNnBny8cLJIdwOG8017qLMJ16K2swujguCPMJHSwHr+2lHsaZcKZau2wJ0Jrw/a45lGl8GjCilIknjaRGRu0Rkr4js7e/vL+jE50KH6R/4tYtWAdCewU+glIoXMZsND77czbMnBvm/N22hsTrzw93yEzza3ktbQyW2Ijk6U0xDZaYRWFFDOodAU67kFAQi8riIHErzunU+JpgOpdTdSqldSqldTU1NCzWNFDr6JmisdnNRax1VLvu0zN5Efnywh9f99S/ipY1nwnggzBd/+ioXtNTynhyVN7et8lLpshOMxIpmFoI0piHTR1BZJg/OGu0s1pQ5OZdsSqkbZvG5XUCiB7PVHCPD+CBQJyIOUytI3L9kON4/wcbmKmw2YduqWg51p9cIHj7UQzSm6B8P0lwzs/j5f378GP0TQe6+Y1fOUEaH3cbOtjqe7hgsWjIZpDcNOWyC21EezlW7TahxO7RpSFO2FOuX+iBwu4i4RWQdsAl4AXgR2GRGCLkwHMoPKiMA/QngHebxdwIPFGluRUEpRUffBBubqwGjccrh7jGiMTVtv3A0xq+ODQAwGZxZ9vGR3jG++cwpbr+0jR2r6/I6ZtcawzxUTEFgaQRWhU4rjHSxZgrPBm+FU2sEmrJlruGjvy4iZ4ErgJ+KyCMASql24D7gMPAw8BGlVNRc7X8UeAR4FbjP3BfgT4E/FpEODJ/BN+Yyt/mmfyLIWCDChiZDEGxvqcUfjnIyqX/wvtPDCSaU1C5imYjFFJ/+0SG8Hgcff9N5eR935cZGADYvr8n7mJnicdpxOWxxH8FEMHd3slLj4zedx52vW7vQ09BoisKcvHlKqR8BP8qw7YvAF9OMPwQ8lGb8BEZUUUliOYotjWDbKqOFYnv3aHwM4MnXppzbk2naSWbiW8+eYu/pYf7+nRdRX5V/isXudQ08/LGrOa+IggDMUtT+qfDRcnEUW9y6I2PsgkZT8pSHEXcRcLzfWPlbD/2NzdW4HDbak/wETxzpi5dd9uVpGjo9OMnfPnyUa89r4u0Xz/yBtGWFt+hmGm+FY1pCWWWZCQKNppzRgqBAHO+boMpljzdXcdptnL+ihkNdU5FDvaMBjvSO8+YLVgKkbTCfTCym+NMfHMRhE/7qbRcsWru7oRFMOYuryySrWKNZCmhBUCA6+ibY0Fw97UG9dVUt7d1j8WJsTx7tA+AWUxDkYxr67+dP89yJIT51y/nxMtOLkRqPY1pmcT69CDQazeJAC4ICcbx/go1N1dPGtrd4GfWHOTvsB+DJo/2srPWwvcWLy26Lx9tnonPIx1/97AhXb2pMW09oMWH09bVMQ5G8exFoNJqFRwuCAjARjNAzGmBD83RBYNUAau8eIxSJsadjgGvPa0JEqHTbs2oESik++cNXEOCv337hojUJWSQ6i33BaNkkk2k0SwG9bCsAVunpDUkawZYVNdhtQnv3KLUVTiaCEa49z2jWXuVyZA0ffeb4IHs6BvjCrdtoqVu8JiELr8fBeEL4qNYINJrSQf9aC0By6KiFx2lnY1O1oRFEYzjtEo/rr8qhEfSOGuUnXr+5uUizLizeCifBSAxfKEIwEiu78FGNppzRv9YCcLx/AodN0mbvbmvxsufYAF3DfnataYivlKvcjnhLx3RYEUXVntL4L/Ka87QEmDYNaTSlg/YRFICOvgnWLKvEaU+9nNtX1dI3HuTouXGu2zJVIK/a7cgaPhpv91giYZhWYbYeUxBo05BGUzpoQVAAOvonUsxCFlaGMRD3D4CxYs6WUDYeiOBy2HA7SkMQWKWaLUGgE8o0mtJBC4I5Eo7GODPoyygItpqCoKWugk0J+1Tl1AjC1JTQw9SqQNo7aoTK6oQyjaZ0KJ0nzSLl9OAkkZhKiRiyqPE42b2ugUvX1k8LAa1yOfBliRqaCERKxj8AU81p4hqBTijTaEoG/WudI5kihhK570NXpIxVuR1Zy1CXWgimVYpa+wg0mtJDm4bmSEeGHIJcVLnshKKxjA3uxwOlJQi8nmSNQJuGNJpSQQuCOXK8f5KVtZ4Zx81b+2cyD00EI/FVdilQ6bJjt0mCj6B05q7RLHW0IJgjiV3JZoIVFprJYVxqpiERocbjYNhnZBfrhDKNpnTQgmAOdI/4OdI7xtaV3tw7JzGlEaT3E5SasximzEOgG71rNKWEFgRz4KtPdgDwvivWzPjYKjOqJpNGMB6MUO12pt22WLFyCapcdmy2xV0kT6PRTKEFwSzpGvHzvRc7eeeu1bTWz7wxfFwjSBM5FIxECUViJeUjgCmNQCeTaTSlhRYEWfidb+/lW8+cSrvtK08Y2sBHrts4q8+2omrSaQQTZoOXUvIRwFQIaanNW6NZ6uhfbAYi0RiPHT7H46+eo6Wughu2Lo9vOzvs4/t7O/mNS1fPukR0dZaooXjBuRJ7oMY1Ah06qtGUFFojyMCYuSq3ifCH9+7ntXPj8W1feaIDQWatDcCUaShdKWqr5WPJOYvN7GIdMaTRlBZaEGTAasT+JzduptLt4Le/tZfhyRCdQz6+v/cst+9ePacewlb4aLp2lZZGUGo+Amu+VVoj0GhKCi0IMjBqCoLzltdw9/suoXcswIf/+yX++fFj2GzC7107e20AjPBKkfQageUjqCm1qCGP1gg0mlJEC4IMWIKgtsLJzrZ6/ubtF/DciSF+sO8s79ndxopaz5w+X0SMdpVpooZKrSmNRdw0pAvOaTQlxZwEgYi8U0TaRSQmIrsSxt8oIi+JyCvmv9cnbLvEHO8QkS+JWZJTRBpE5DEROWb+Wz+Xuc2VREEA8Os7W/n96zfSWO3iw9duKMg5MrWrHC9RZ3HcNFRi89Zoljpz1QgOAW8Dfpk0PgC8VSl1AXAn8F8J274G/A6wyXzdZI5/Avi5UmoT8HPz/YJhCYL/3979xshVnXcc//52h12za1wnAYyDcU0FBUEgJt0QrNI25V9JFBVDSJu0ErxAdaskEm3TJiCkiFRUgr4gSRUUySJtSRMVUlpKxJ+6QC2hNoXEBpMaXAeTULHUYEPBa6931zs7T1/cM/bdYdbL7njunen8PtLIc8+9O/vIvr7PnHPuPU/9Wy7AF644i6duvpQVy1rrDdTNVcD+8NBQt/UI0tCQaxGYdZeWEkFE7IiInU3an42I/0mbzwPHSxqUtBJYFhFPRUQA3wbWp+OuAu5J7+/JtZeisUdQV2lSjnKxsqWom90+Ok2lTwxWumvkrv5ksR8oM+suRVxpPgk8ExFTwKnAaG7faGoDWBERu9P714AVlGhsYpqBSh9L2rhmztBAf/O7htI6Q/lCNt2gnjS7bUjLrNfN+z9W0uPAKU123RIRD87zs+cCdwBXLCSoiAhJcZTP3QBsAFi9evVCPvpd2zcx/Y7ewLG2dLDCa2OT72jf32Urj9aduvx4/uyqc/n4eSvLDsXMFmDeq01EXLaYD5a0CngAuC4iXkrNrwKrcoetSm0Ar0taGRG70xDSnqPEtBHYCDAyMjJnwmjF2GT7E8HQXENDXVaUpk4S161bU3YYZrZAbRkakrQceBi4KSL+vd6ehn7GJF2U7ha6Dqj3Kr5PNrFM+vOovY12K6ZHMMfQUJcVpTGz7tbq7aNXSxoF1gEPS9qUdn0eOAP4sqRt6XVy2vdZ4G5gF/AS8Ghqvx24XNKLwGVpuzRFJIKhgeY9gm4rU2lm3a2lq01EPEA2/NPYfhtw2xw/swX4QJP2N4FLW4nnWNo3Mc2ZJ5/Q1t8xPFjh4KEZarWYtX7/gakqa04cbuvvNjOr6677Ewu07+A0y9o8PFO/3/7g9OzhIfcIzKxITgRN1GrB/qlqIUNDAAcbhocOTE17jsDMCuNE0MT+ySoRs58qbof6t/58cZrpmRqT0zX3CMysME4ETcz1VPGxVi/gki9gP96l6wyZWfdyImiiqETQrEfQrUVpzKx7ORE0MTZZUI+gSbnKelJo90S1mVmdE0ETh3sEQ+1/oAzgQK4mwZF6xd1VlMbMupcTQRPFzRG8866hAx4aMrOCORE0cbgWwZL2JoLhZnMEniw2s4I5ETSxbyKrBzDU5iLs9SLv+XKV3VqUxsy6lxNBE/V1htpdD6DS38dgpa9hsjjrjbhHYGZFcSJooogF5+qGByuzhoYOTFaRaHtvxMyszomgibGJ6bY/VVw3PNg/64GysbTOULdVJzOz7uVE0EShPYKBhh7BVJUTPCxkZgVyImhirOChoVlzBKlesZlZUZwImih+jmD2A2WeKDazIjkRNIgIxiarLDu+mIvx8ED/rAfK9k9VWdrm5xfMzPKcCBocmKoyU4tCewTjs+4amvYcgZkVyomgQVHLS9QND8wuYO+hITMrmhNBg8ITQeoRRATgyWIzK54TQYPD6wwVmAiqteDQTI2ZWjB+aMY9AjMrlK84DcZKGBqCbL2hSn8N8DpDZlYsX3EaFD00VC9OMz5Vpb8ve5rYicDMiuQrToOxiewOnqISQX0YaPxQlb60rISL0phZkZwIGuybmKZP2dIPRRg6PDRUBVIicI/AzArkK06DfWnBub6+YhZ9O9wjmJqhlu4c8mSxmRWppbuGJH1K0vOSapJGmuxfLemApD/JtV0paaekXZJuyrWfLunp1H6fpIFWYlusIpeXgCPlKsenqocXn/McgZkVqdXbR7cD1wBPzrH/TuDR+oakfuAu4GPAOcBnJJ2Tdt8BfDUizgDeAm5oMbZFKToRHJkjmDlSr9g9AjMrUEuJICJ2RMTOZvskrQd+Bjyfa74Q2BURP42IQ8C9wFXKFt+/BLg/HXcPsL6V2Bar6EQwPHhkjqDeI/AcgZkVqS0PlElaCnwJ+ErDrlOBV3Lbo6ntfcDbEVFtaC9ckUVp4EgB+/FDVfanHkFRE9VmZvAuJoslPQ6c0mTXLRHx4Bw/divZMM+BdlTakrQB2ACwevXqY/rZRfcIBit99PeJ8akqk9M1hgf6Dz9PYGZWhHkTQURctojP/QhwraS/AJYDNUmTwFbgtNxxq4BXgTeB5ZIqqVdQb58rpo3ARoCRkZFYRHxzfW7hiUASQwP9jE/NMHFoxsNCZla4tlx1IuJX6u8l3QociIhvSKoAZ0o6nexC/2ngdyIiJG0GriWbN7gemKu30TYT0zNUa8GygusBLE0Lzx30OkNmVoJWbx+9WtIosA54WNKmox2fvu1/HtgE7AC+FxH1yeQvAX8saRfZnMG3WoltMYpeXqJuaCArYO+iNGZWhpa+fkbEA8AD8xxza8P2I8AjTY77KdldRaUpKxEsHcwK2LsojZmVwctQ5+w7WFaPoHL49lEPDZlZ0ZwIcsrqEQwPVg4/UObJYjMrmhNBTnmJoJ/xqSr7p6peXsLMCudEkFNqjyANDXmOwMyK5kSQMzYxjVT8om/DA/3878FDRHh5CTMrnhNBzr6JaZYOVgpbgrpueLBCWoHaRWnMrHBOBDljk9XCh4Vg9mqj7hGYWdGcCHKKXl6ibii3yJznCMysaE4EOWUlgvpS1OAegZkVz4kgp7REkOsR+IEyMyuaE0FOeT0CJwIzK48TQU4nDA35gTIzK5oTQTI5PcOhaq3Q6mR1+R7BsHsEZlYwJ4Kk/lRxKYkgzREsOa6P4/r9T2JmxfJVJylreQk4MjTkh8nMrAxOBMlYiYmg/hyB5wfMrAxOBEmZPYL+PnH8cf2+Y8jMSuFEkJSZCCAbHnIiMLMyOBEk5SeCip8qNrNS+MoDzNSCx154naGBfpaVdDH+0984i5OWDpbyu82stzkRAHdt3sUPXnqTOz55HpWSbt/8xPnvL+X3mpn1/NDQD156g689/hOuvuBUfmvktLLDMTMrXE8ngr37p7jx3m2sOXGY29Z/AKnYgjRmZp2gZ4eGZmrBH923jbGJaf72hgu9tIOZ9ayevfrdtXkX/7brDW6/5jzOPmVZ2eGYmZWmJ4eG9oxN8o3Nu1i/9v389oc9L2Bmva2lRCDpU5Kel1STNNKw73xJ/5H2/6ekJan9l9L2Lkl/qTQwL+m9kh6T9GL68z2txHY0Jy9bwvd+fx1/fvV5nhcws57Xao9gO3AN8GS+UVIF+A7wBxFxLvBRYDrt/ibwe8CZ6XVlar8JeCIizgSeSNtts/a05Z4XMDOjxUQQETsiYmeTXVcAP46I59Jxb0bEjKSVwLKIeCoiAvg2sD79zFXAPen9Pbl2MzNro3bNEfwiEJI2SXpG0hdT+6nAaO640dQGsCIidqf3rwEr5vpwSRskbZG0Ze/evcc6djOznjLv2Iikx4FTmuy6JSIePMrnXgx8GDgIPCFpK7Dv3QQVESEpjrJ/I7ARYGRkZM7jzMxsfvMmgoi4bBGfOwo8GRFvAEh6BPgQ2bzBqtxxq4BX0/vXJa2MiN1pCGnPIn6vmZktULuGhjYB50kaShPHvwa8kIZ+xiRdlO4Wug6o9yq+D1yf3l+fazczszZq9fbRqyWNAuuAhyVtAoiIt4A7gR8B24BnIuLh9GOfBe4GdgEvAY+m9tuByyW9CFyWts3MrM2U3bzTvUZGRmLLli1lh2Fm1nEkbY2IkfmO68kni83M7AgnAjOzHudEYGbW47p+jkDSXuC/F/njJwJvHMNwiuCYi9GNMUN3xu2Y2+fnI+Kk+Q7q+kTQCklb3s1ESidxzMXoxpihO+N2zOXz0JCZWY9zIjAz63G9ngg2lh3AIjjmYnRjzNCdcTvmkvX0HIGZmblHYGbW83oyEUi6UtLOVC6zrZXQWiHpryTtkbQ911ZYSc/FkHSapM2SXkhlSm9M7R0bt6Qlkn4o6bkU81dS++mSnk7nyX2SBsqOtZGkfknPSnoobXd0zJJeTqVqt0nakto69twAkLRc0v2S/kvSDknrOj3mheq5RCCpH7gL+BhwDvAZSeeUG9Wc/oYjpTzrCi3puQhV4AsRcQ5wEfC59PfbyXFPAZdExAeBtcCVki4C7gC+GhFnAG8BN5QY41xuBHbktrsh5l+PiLW52y87+dwA+DrwzxFxNvBBsr/vTo95YSKip15kK6Vuym3fDNxcdlxHiXcNsD23vRNYmd6vBHaWHeM88T8IXN4tcQNDwDPAR8geGKo0O2864UVWz+MJ4BLgIUBdEPPLwIkNbR17bgA/B/yMNJ/aDTEv5tVzPQKy0piv5Lbz5TK7wbsu6Vk2SWuAC4Cn6fC40xDLNrKCSI+RLZH+dkRU0yGdeJ58DfgiUEvb76PzYw7gXyRtlbQhtXXyuXE6sBf46zQEd7ekYTo75gXrxUTw/0ZkX0c68rYvSUuBfwD+MCLG8vs6Me6ImImItWTfsi8Ezi45pKOS9AlgT0RsLTuWBbo4Ij5ENjT7OUm/mt/ZgedGhay64jcj4gJgnIZhoA6MecF6MRG8CpyW286Xy+wGr6dSnnRqSU9Jx5Elge9GxD+m5o6PGyAi3gY2kw2rLE8V9qDzzpNfBn5T0svAvWTDQ1+ns2MmIl5Nf+4BHiBLup18bowCoxHxdNq+nywxdHLMC9aLieBHwJnp7ooB4NNkZTK7RUeX9EwlSL8F7IiIO3O7OjZuSSdJWp7eH082p7GDLCFcmw7rqJgj4uaIWBURa8jO4X+NiN+lg2OWNCzphPp74ApgOx18bkTEa8Arks5KTZcCL9DBMS9K2ZMUZbyAjwM/IRsHvqXseI4S598Bu4Fpsm8mN5CNAz8BvAg8Dry37DgbYr6YrJv8Y7IypdvS33fHxg2cDzybYt4OfDm1/wLwQ7Kyqn8PDJYd6xzxfxR4qNNjTrE9l17P1//vdfK5keJbC2xJ58c/Ae/p9JgX+vKTxWZmPa4Xh4bMzCzHicDMrMc5EZiZ9TgnAjOzHudEYGbW45wIzMx6nBOBmVmPcyIwM+tx/wdpNokIb2lwBQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "i_update = 0\n",
    "state = envs.reset()\n",
    "early_stop = False\n",
    "\n",
    "while frame_idx < max_frames and not early_stop:\n",
    "    i_update += 1\n",
    "    \n",
    "    log_probs = []\n",
    "    values    = []\n",
    "    states    = []\n",
    "    actions   = []\n",
    "    rewards   = []\n",
    "    masks     = []\n",
    "    entropy = 0\n",
    "\n",
    "    for _ in range(num_steps):\n",
    "        state = torch.FloatTensor(state).to(device)\n",
    "        dist, value = model(state)\n",
    "\n",
    "        action = dist.sample()\n",
    "        next_state, reward, done, _ = envs.step(action.cpu().numpy())\n",
    "        reward = expert_reward(state, action.cpu().numpy())\n",
    "        \n",
    "        log_prob = dist.log_prob(action)\n",
    "        entropy += dist.entropy().mean()\n",
    "        \n",
    "        log_probs.append(log_prob)\n",
    "        values.append(value)\n",
    "        rewards.append(torch.FloatTensor(reward).to(device))\n",
    "        masks.append(torch.FloatTensor(1 - done).unsqueeze(1).to(device))\n",
    "        \n",
    "        states.append(state)\n",
    "        actions.append(action)\n",
    "        \n",
    "        state = next_state\n",
    "        frame_idx += 1\n",
    "        \n",
    "        if frame_idx % 1000 == 0:\n",
    "            test_reward = np.mean([test_env() for _ in range(10)])\n",
    "            test_rewards.append(test_reward)\n",
    "            plot(frame_idx, test_rewards)\n",
    "            if test_reward > threshold_reward: early_stop = True\n",
    "            \n",
    "\n",
    "    next_state = torch.FloatTensor(next_state).to(device)\n",
    "    _, next_value = model(next_state)\n",
    "    returns = compute_gae(next_value, rewards, masks, values)\n",
    "\n",
    "    returns   = torch.cat(returns).detach()\n",
    "    log_probs = torch.cat(log_probs).detach()\n",
    "    values    = torch.cat(values).detach()\n",
    "    states    = torch.cat(states)\n",
    "    actions   = torch.cat(actions)\n",
    "    advantage = returns - values\n",
    "    \n",
    "    if i_update % 3 == 0:\n",
    "        ppo_update(4, mini_batch_size, states, actions, log_probs, returns, advantage)\n",
    "    \n",
    "    \n",
    "    expert_state_action = expert_traj[np.random.randint(0, expert_traj.shape[0], 2 * num_steps * num_envs), :]\n",
    "    expert_state_action = torch.FloatTensor(expert_state_action).to(device)\n",
    "    state_action        = torch.cat([states, actions], 1)\n",
    "    fake = discriminator(state_action)\n",
    "    real = discriminator(expert_state_action)\n",
    "    optimizer_discrim.zero_grad()\n",
    "    discrim_loss = discrim_criterion(fake, torch.ones((states.shape[0], 1)).to(device)) + \\\n",
    "            discrim_criterion(real, torch.zeros((expert_state_action.size(0), 1)).to(device))\n",
    "    discrim_loss.backward()\n",
    "    optimizer_discrim.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_env(True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pytorch4]",
   "language": "python",
   "name": "conda-env-pytorch4-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}