dqn-her.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DQN HER Implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import namedtuple\n",
    "from itertools import count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import torchvision.transforms as T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up matplotlib\n",
    "is_ipython = 'inline' in matplotlib.get_backend()\n",
    "if is_ipython:\n",
    "    from IPython import display\n",
    "\n",
    "plt.ion()\n",
    "\n",
    "# if gpu is to be used\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bit flipping environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BitFlipEnv():\n",
    "    \n",
    "    def __init__(self, n = 8):\n",
    "        '''\n",
    "        Setup the environment with a init state and target state\n",
    "        The init and target stae should not be equal\n",
    "        '''\n",
    "        self.n = n\n",
    "        self.init_state = torch.randint(2, size=(n,))\n",
    "        self.target_state = torch.randint(2, size=(n,))\n",
    "        while np.array_equal(self.init_state, self.target_state):\n",
    "            self.target_state = torch.randint(2, size=(n,))\n",
    "        self.curr_state = self.init_state.clone()\n",
    "        \n",
    "    def step(self, action):\n",
    "        '''\n",
    "        Take a step, i.e. flip the bit specified by the position action\n",
    "        Return the next state and the reward \n",
    "        Reward is 0 if the target state is reacher\n",
    "        Otherwise reward is -1\n",
    "        '''\n",
    "        self.curr_state[action] = 1 - self.curr_state[action]\n",
    "        if np.array_equal(self.curr_state, self.target_state):\n",
    "            return self.curr_state.clone(), 0\n",
    "        else:\n",
    "            return self.curr_state.clone(), -1\n",
    "        \n",
    "    def reset(self):\n",
    "        '''\n",
    "        Reset the bit flip environment\n",
    "        '''\n",
    "        self.init_state = torch.randint(2, size=(self.n,))\n",
    "        self.target_state = torch.randint(2, size=(self.n,))\n",
    "        while np.array_equal(self.init_state, self.target_state):\n",
    "            self.target_state = torch.randint(2, size=(self.n,))\n",
    "        self.curr_state = self.init_state.clone()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = BitFlipEnv(n=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial state: tensor([ 0.,  0.,  0.,  0.,  0.,  1.,  1.,  0.,  0.,  1.])\n",
      "Target state: tensor([ 0.,  0.,  0.,  0.,  0.,  0.,  1.,  1.,  1.,  0.])\n",
      "State, reward after taking action 9: tensor([ 0.,  0.,  0.,  0.,  0.,  1.,  1.,  0.,  0.,  0.]) -1\n",
      "<class 'torch.Tensor'>\n"
     ]
    }
   ],
   "source": [
    "#testing the bit flip env\n",
    "print('Initial state:', env.init_state)\n",
    "print('Target state:', env.target_state)\n",
    "curr_state, reward = env.step(9)\n",
    "print('State, reward after taking action 9:', curr_state, reward)\n",
    "print(type(curr_state))\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Replay Memory\n",
    "\n",
    "HER Capable replay memory, i.e. additionally store the goal state for each transition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "Transition = namedtuple('Transition', \n",
    "                       ('state', 'action', 'next_state', 'reward', 'goal'))\n",
    "\n",
    "class ReplayMemory(object):\n",
    "    \n",
    "    def __init__(self, capacity = 1e5):\n",
    "        self.capacity = capacity\n",
    "        self.memory = []\n",
    "    \n",
    "    def push(self, *args):\n",
    "        \"\"\"Saves a transition which should contain:\n",
    "        - current state\n",
    "        - action taken\n",
    "        - next state\n",
    "        - reward obtained\n",
    "        - goal state\"\"\"\n",
    "        self.memory.append(Transition(*args))\n",
    "        if len(self.memory) > self.capacity:\n",
    "#             print('!!!!!memory capacity exceeded!')\n",
    "            del self.memory[0]\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        \"\"\"\n",
    "        Returns batch_size number of samples from the replay memory\n",
    "        \"\"\"\n",
    "        return random.sample(self.memory, batch_size)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.memory)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Definition\n",
    "A feedforward NN with 1 hidden layer of size 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_BITS = 8\n",
    "HIDDEN_SIZE = 256 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FNN(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(FNN, self).__init__()\n",
    "        self.ln1 = nn.Linear(NUM_BITS*2, HIDDEN_SIZE)\n",
    "        self.ln2 = nn.Linear(HIDDEN_SIZE, NUM_BITS)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.ln1(x))\n",
    "        x = self.ln2(x)\n",
    "        return x\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyper parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128 # batch size for training\n",
    "GAMMA = 0.999 # discount factor\n",
    "EPS_START = 0.95 # eps greedy parameter\n",
    "EPS_END = 0.05\n",
    "TARGET_UPDATE = 50 # number of epochs before target network weights are updated to policy network weights\n",
    "steps_done = 0 # for decayin eps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize Model\n",
    "Initialize a policy network and a target network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "policy_net = FNN().to(device)\n",
    "target_net = FNN().to(device)\n",
    "target_net.load_state_dict(policy_net.state_dict())\n",
    "target_net.eval()\n",
    "\n",
    "optimizer = optim.RMSprop(policy_net.parameters())\n",
    "memory = ReplayMemory(1e6) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def select_action(state, goal, greedy=False):\n",
    "    '''\n",
    "    Select an action given the state and goal acc to policy_net\n",
    "    - use eps_greedy policy when greedy=False\n",
    "    - use greedy policy when greedy=True\n",
    "    Returns action taken which is from range(0, n-1)\n",
    "    '''\n",
    "    global steps_done\n",
    "    sample = random.random()\n",
    "    state_goal = torch.cat((state, goal))\n",
    "\n",
    "    eps_threshold = EPS_END + (EPS_START - EPS_END) * \\\n",
    "        math.exp(-1. * steps_done / EPS_DECAY)\n",
    "    steps_done += 1\n",
    "#     if steps_done % 1000 == 0:\n",
    "#         print('Steps done: {} Epsilon threshold: {}'.format(steps_done, eps_threshold))\n",
    "    if greedy:\n",
    "        with torch.no_grad():\n",
    "            return policy_net(state_goal).argmax().view(1,1)\n",
    "    if sample > eps_threshold:\n",
    "        with torch.no_grad():\n",
    "            return policy_net(state_goal).argmax().view(1,1)\n",
    "    else: \n",
    "        return torch.tensor([[random.randrange(NUM_BITS)]], device=device, dtype=torch.long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize_model():\n",
    "    '''\n",
    "    optimize the model, i.e. perform one step of Q-learning using BATCH_SIZE number of examples\n",
    "    '''\n",
    "    if len(memory) < BATCH_SIZE:\n",
    "        return\n",
    "    transitions = memory.sample(BATCH_SIZE)\n",
    "    batch = Transition(*zip(*transitions))\n",
    "    \n",
    "    non_final_mask = torch.tensor(tuple(map(lambda s: s is not None, \n",
    "                                           batch.next_state)), device=device, dtype=torch.uint8)\n",
    "    non_final_next_states = torch.stack([s for s in batch.next_state \n",
    "                                      if s is not None])\n",
    "    \n",
    "    # extract state, action, reward, goal from randomly sampled transitions\n",
    "    state_batch = torch.stack(batch.state)\n",
    "    action_batch = torch.cat(batch.action)\n",
    "    reward_batch = torch.cat(batch.reward)\n",
    "    goal_batch = torch.stack(batch.goal)\n",
    "    \n",
    "    # concatenate state and goal for network input\n",
    "    state_goal_batch = torch.cat((state_batch, goal_batch), 1)\n",
    "    non_final_next_states_goal = torch.cat((non_final_next_states, goal_batch), 1)\n",
    "    \n",
    "    # get current state action values \n",
    "    state_action_values = policy_net(state_goal_batch).gather(1, action_batch)\n",
    "    \n",
    "    # get next state values according to target_network\n",
    "    next_state_values = torch.zeros(BATCH_SIZE, device=device)\n",
    "    next_state_values[non_final_mask] = target_net(non_final_next_states_goal).max(1)[0].detach()\n",
    "    \n",
    "    # calculate expected q value of current state acc to target_network\n",
    "    expected_state_action_values = (next_state_values * GAMMA) + reward_batch.float()\n",
    "    \n",
    "    # find huber loss using curr q-value and expected q-value\n",
    "    loss = F.smooth_l1_loss(state_action_values, expected_state_action_values.unsqueeze(1))\n",
    "    \n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    for param in policy_net.parameters():\n",
    "        param.grad.data.clamp_(-1, 1)\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# named tuple for storing the hindsight transitions, i.e. without goal state\n",
    "HindsightTransition = namedtuple('HindsightTransition', \n",
    "                       ('state', 'action', 'next_state', 'reward'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1200.0\n"
     ]
    }
   ],
   "source": [
    "CHECK_RATE = 100 # evaluate success on the last 100 episodes\n",
    "MODEL_PATH = '_her_policy_net.pt'\n",
    "WEIGHTS_PATH = '_her_policy_net_weights.pt'\n",
    "FIGURE_PATH = '_her.png'\n",
    "SAVE_MODEL = True\n",
    "LOAD_MODEL = False\n",
    "num_episodes = 3000\n",
    "EPS_DECAY = num_episodes * NUM_BITS * 0.05 # decay rate\n",
    "print(EPS_DECAY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "success rate for last 100 episodes after 0 episodes of training: 0.0%\n",
      "success rate for last 100 episodes after 100 episodes of training: 3.0%\n",
      "success rate for last 100 episodes after 200 episodes of training: 14.000000000000002%\n",
      "success rate for last 100 episodes after 300 episodes of training: 31.0%\n",
      "success rate for last 100 episodes after 400 episodes of training: 53.0%\n",
      "success rate for last 100 episodes after 500 episodes of training: 63.0%\n",
      "success rate for last 100 episodes after 600 episodes of training: 76.0%\n",
      "success rate for last 100 episodes after 700 episodes of training: 77.0%\n",
      "success rate for last 100 episodes after 800 episodes of training: 91.0%\n",
      "success rate for last 100 episodes after 900 episodes of training: 87.0%\n",
      "success rate for last 100 episodes after 1000 episodes of training: 88.0%\n",
      "success rate for last 100 episodes after 1100 episodes of training: 93.0%\n",
      "success rate for last 100 episodes after 1200 episodes of training: 95.0%\n",
      "success rate for last 100 episodes after 1300 episodes of training: 96.0%\n",
      "success rate for last 100 episodes after 1400 episodes of training: 98.0%\n",
      "success rate for last 100 episodes after 1500 episodes of training: 95.0%\n",
      "success rate for last 100 episodes after 1600 episodes of training: 95.0%\n",
      "success rate for last 100 episodes after 1700 episodes of training: 99.0%\n",
      "success rate for last 100 episodes after 1800 episodes of training: 99.0%\n",
      "success rate for last 100 episodes after 1900 episodes of training: 97.0%\n",
      "success rate for last 100 episodes after 2000 episodes of training: 97.0%\n",
      "success rate for last 100 episodes after 2100 episodes of training: 99.0%\n",
      "success rate for last 100 episodes after 2200 episodes of training: 99.0%\n",
      "success rate for last 100 episodes after 2300 episodes of training: 98.0%\n",
      "success rate for last 100 episodes after 2400 episodes of training: 98.0%\n",
      "success rate for last 100 episodes after 2500 episodes of training: 98.0%\n",
      "success rate for last 100 episodes after 2600 episodes of training: 98.0%\n",
      "success rate for last 100 episodes after 2700 episodes of training: 99.0%\n",
      "success rate for last 100 episodes after 2800 episodes of training: 100.0%\n",
      "success rate for last 100 episodes after 2900 episodes of training: 98.0%\n"
     ]
    }
   ],
   "source": [
    "env = BitFlipEnv(NUM_BITS)\n",
    "success = 0\n",
    "episodes = [] # every 100 episodes\n",
    "success_rate = [] # append success rate of last 100 episodes\n",
    "# train the network\n",
    "for i_episode in range(num_episodes):\n",
    "    env.reset()\n",
    "    state = env.init_state\n",
    "    goal = env.target_state\n",
    "    transitions = []\n",
    "    episode_success = False\n",
    "    # for bit length\n",
    "    for t in range(NUM_BITS):\n",
    "        \n",
    "        if episode_success:\n",
    "            continue\n",
    "        \n",
    "        action = select_action(state, goal)\n",
    "        next_state, reward = env.step(action.item())\n",
    "        reward = torch.tensor([reward], device=device)\n",
    "        \n",
    "        # add transition to replay memory\n",
    "        memory.push(state, action, next_state, reward, goal)\n",
    "        \n",
    "        # store transition without goal state for hindsight \n",
    "        transitions.append(HindsightTransition(state, action, next_state, reward))\n",
    "\n",
    "        state = next_state\n",
    "        \n",
    "        optimize_model()\n",
    "        if reward == 0:\n",
    "            if episode_success:\n",
    "                continue\n",
    "            else:\n",
    "                episode_success = True\n",
    "                success += 1\n",
    "    \n",
    "    # add hindsight transitions to the replay memory\n",
    "    if not episode_success:\n",
    "        # failed episode store the last visited state as new goal\n",
    "        new_goal_state = state.clone()\n",
    "        if not np.array_equal(new_goal_state, goal):\n",
    "            for i in range(NUM_BITS):\n",
    "                # if goal state achieved\n",
    "                if np.array_equal(transitions[i].next_state, new_goal_state):\n",
    "                    memory.push(transitions[i].state, transitions[i].action, transitions[i].next_state, torch.tensor([0]), new_goal_state)\n",
    "                    optimize_model()\n",
    "                    break\n",
    "\n",
    "                memory.push(transitions[i].state, transitions[i].action, transitions[i].next_state, transitions[i].reward, new_goal_state)\n",
    "                optimize_model()\n",
    "\n",
    "    # update the target networks weights \n",
    "    if i_episode % TARGET_UPDATE == 0:\n",
    "        target_net.load_state_dict(policy_net.state_dict())\n",
    "\n",
    "    if i_episode % CHECK_RATE == 0:\n",
    "        print('success rate for last {} episodes after {} episodes of training: {}%'.format(CHECK_RATE, i_episode, success/CHECK_RATE * 100))\n",
    "        success_rate.append(success/CHECK_RATE)\n",
    "        episodes.append(i_episode)\n",
    "        success = 0\n",
    "\n",
    "episodes.append(num_episodes)\n",
    "success_rate.append(success/CHECK_RATE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot the success rate across number of episodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8XVW5//HPk7RpOqRN5ylJZyhTSaEtUCwg8EMmoYLMM/zkihe9etUryhURvD8Fr3r1ygWrQhkFlBmB4lUKhZbSliYd6TwkHZO2ScekGZ7fH3snnIaT5KTNyTkn+b5fr/PKHtbZ51nZyX7OXmvvtc3dERERAUhLdAAiIpI8lBRERKSekoKIiNRTUhARkXpKCiIiUk9JQURE6ikpSMowszvMbJuZ7TWzvomOR2JjZuvN7NxG1k0xsxVtHZM0Tkmhg4j2j2lmN5vZ+w3KHAgPunWv30aUrQmX7TazQjO7uA3j7wz8EjjP3Xu4+462+uz2INx/bmbfbbC82MzOauG2zMx+YmabzKzczGaa2XGHE5e7z3L3oyO23WgCkbahpCANfTE86Na97oxYN8fdewDZwP8Az5pZdrwDMrNOwEAgE1h6GO83M9PfOuwEvmdmPY9wO1cAtwJTgD7AHODJI9ymJAn9o0iLuXstwUGgOzAmWhkzOyv8FvoDMysNvwFeF7G+i5n9p5ltDJuEHjGzrg3e+z0z2xp+Vl0TQ5mZ/SMsN9nM5oXfVueZ2eSI7c80s/8wsw+A/cDIcNlPzGx2eMbzmpn1NbOnw7OfeWY2PGIbvzazonDdAjObErHuXjN73syeMLM9ZrbUzCZErM81sxfNrMTMdtSdcYXrbjWz5Wa2y8xmmNmwxn7XZnZJuO2yMP5jItatN7PvmNmi8HfwnJllNrHrlhMcwL/VRJlYjADed/e17l4DPAUc28x7JprZsrDOj9XFWbevw+kngTzgtXD//JuZZZrZU+HvsCzcRwOPMH5pgpKCtJiZpQO3AFXAhiaKDgL6AUOBm4BpZlbXVPAAcBSQD4wOy9zT4L19gGEE30rrmiey3f1sM+sD/BX4DdCXoGnpr3ZoX8MNwO1AVkScV4fLhwKjCA6Sj4WftRz4UcT754Xx9QGeAf7c4KB7CfAswZnTq0BdU1s68Hr4mcPDz3o2XDcV+AFwGdAfmAX8Kdovz8yOCtd9Myz7BsEBMyOi2JXA+QQH6nHAzdG2FeGHwLfC31/Dz7s2PPA29soLiz4LjDazo8JmvZuAt5r53OuALxD8zo8C/r1hAXe/AdjIp2erD4bb7gXkEuznrwIHmvksORLurlcHeAHrgb1AWcRrP8E3vqbKfCVcdzNQHS6rIvjHvLKJzzsrLN89YtnzBAclA/YBoyLWnQasi3jvQSAzYv1wwIFO4fwNwEcNPnMOcHM4PRO4r8H6mcDdEfO/AN6MmP8iUNBEnXYBJ4bT9wL/G7HuWOBARF1K6mJtsI03gdsi5tPC/TAsStkfAs83KLsJOCtif10fsf5B4JFGYr+5bl+H++GBcLq4bnst+FvKAH4d7o9qYB0wopm/va9GzF8IrInY18UNyp4bMX8rMBsYl+j/oY7y0plCxzLV3bPrXsDXmivj7r+PWPdh+L7eBN+Mp0R5f6Rd7r4vYn4DMITgW283YEHdt1CCb5r9I8qWuHtFE9sewmfPUjYQfCuvUxTlfdsipg9Eme9RN2Nm3w6becrDGHsRnPnU2RoxvR/IDPs/coEN7l4d5fOHAb+OqPdOgiQ5NErZQ+roQbNdUYOyDWPoQfPuAe4ws0ExlI3mR8BEgnpmAj8G/mFm3Zp4T+S+qPs7iMWTwAyC/qvNZvZgeHYicaKkIC3m7nsJEsoNZja+iaK9zax7xHwesBkoJTgAHxeRfHp50Ild/zHNhLGZ4AAbKY/gm3Ss22hU2H/wPYLmmd5hMiwnOIA3pwjICxNEtHX/1CDxdnX32VHKHlJHMzOCA/GmKGVj5u6fAC8SNGPVM7Pr7NArzxq+6pqPTgSec/did6929+kEXxSa6lfIjZiu+zuIGl6DWKvc/cfufiwwGbgYuDHWukrLKSnIYfHgktA/cGg/QDQ/NrOM8CB7MfDn8Bvv74FfmdkAADMbamZfaEEIbwBHhe3gnczsKoKD0ustrkx0WQRNIyVAJzO7B4j1qp2PgC3Az8yse9hZenq47hHg+xZewmlmvczsika28zxwkZmdE347/jZQSdCccqR+TNAvVH/1mLs/7YdeedbwtTEsOg+4wswGmlmamd0AdAZWN/F5/2xmOWFfxg+A5xoptw0YWTdjZp83sxPCfprdBE2XNYdbaWmekoI09FqDb4cvNVH2v4ALzWxcI+u3ErTDbwaeJmhX/iRc9z2Cg8iHZrYb+F/g6KhbiSJMShcTHCh3AP8GXOzupbFuoxkzCNr/VxI0d1QQvTkqWmw1BP0Towk6TouBq8J1LxF0sj8b1nsJcEEj21kBXA/8N8HZ1RcJOmEPHnatPt32Oj69gqylHgAKgQKCPqZvAZe7e1kT73kGeBtYG75+0ki5nwL/HjavfYfggoO/ECSE5cC7BFc7SZyYux6yI63PghuinnL3nETHIiKx05mCiIjUi1tSMLNHzWy7mS1pZL2Z2W/MbHV4881J8YpFRERiE88zhekEN9U05gKCu2HHENxg9HAcY5E25u4z1XQkknrilhTc/T2Ca7AbcynwhAc+BLLNbHC84hERkeZFu466rQzl0Ks5isNlWxoWNLPbCc4m6N69+8ljx45tkwBFRJpTWV3L/oPVHDhYw/6DNVRU1dTfbJFuRu/uGfTtnkFGp8R24S5YsKDU3fs3Vy6RSSHaTUBRL4Vy92nANIAJEyb4/Pnz4xmXSFLYXHaAbbubuqn7UyP79aBXt7a90fdgdS2fbN1NTW3zVzB2Tk9jVP8edM1Ib4PI4mfH3koKisrqX4VFZeypCG5c752Rzpk52eTnZZOfm02PLp14Zu5G3lq6lb3unHPMQG6ZPJzTRvUluA+xbZlZU+OU1UtkUijm0Lscc2j8LkeRDuXP84v4/ouLqY7hgAuQ2TmNqflDuWnycI4ZfKQjYzdt+54Knpm7kafnbqRkT2XM70tPM8YOyuLE3OCgOT43m1H9e5CW1vYHyFhUVNWwdHM5CzeGCaC4jKKdwVh8aQZHD+rJReOGkJ/bi/zc3owe0IP0BnU5fXQ/tpQf4KkPN/DM3I38bdk2jhrYg5smD+dL44fSLSORh+Do4nqfggXDEL/u7sdHWXcRcCfB4FinAL9x90nNbVNnCtKe1dY6v/zbSn77zmpOH92X//u5kc0OrFFT4/z9k228+PEmKqtrOXVkH26ePIJzjxlAp/TWa7IoLCpj+uz1vL5oM1U1zllH9+fyk3Lokdn8ga3iYA1LNpdTWFQefLuuDL5dZ3XpxLjcXuTnZnNiTjYn5maTFcP2Wps7bCmvoDDiLGD5lt31SXlwr0zyw2SWn5vNCTm9WnxAr6iq4dXCzUz/YD3LtuymZ2Ynrp6Ux7WT8hjQs0tM2+icnkbnw9ynZrbA3Sc0Wy5eScHM/kQwAmI/glvXf0RwKzzu/kg4jstvCa5Q2g/c4u7NHu2VFKQtVNfU8rdl23hs9npWbtvDN84ew02Th3/mm2Brqqiq4Tt/LuT1RVu4emIu9089vkUHgF37DvLc/CKenLOBTWUHGJrdlRtOG8bVE3PJ7pbR/AaiOFhdy5tLtvDYB+spKCqje0Y6V0zI5cbThjGyfyxj731Wba2ztnRv/TfwgqIyPtm6J6ZmqLbQPSOdcRHNQPm52Qzs2dRjKlrG3Zm/YRfTP1jPW0u3tqjeP5l6PNef2ujjN5qU8KQQL0oKEk879x3k2XkbeWrOBjaXV5DTuys5vbvy4dqd5Odm8+CXx3HUwKxW/9wdeyv5yhPz+XhjGXddMJZ/OmPkYbc7V9fU8r/LtzN99jo+XLuTzM5pfGl80LQ0dlBsTUsleyp5Zu5Gnpq7gZI9lYzo152bThvG5SfnkJXZ+n0XBw4GTTVLNpVTWV3b6tuPRe9uGeTnBU1a8Uz+kbaUH2DGkq0x1/lzY/px3JBeh/VZSgoiLbBs824en72elwuCJpjTR/fl5skjOHvsANIMXinYzI9fW8reymruOGs0//z5UXTp1Dqdpqu37+XW6fPYtruCX12Vz4UntN6V2cu37OaJOevrm5aGZnclLYaTj63lFfVNRDdNHs6ZY/onbdu/xEZJQaQZkU1EH63bSdfO6Vx2UvCNOtrZwI69ldz/+jJeLtjM6AE9eODyEzh52GceYNYis9eU8tUnF5DRKY3f3ziB8Xm9j2h7jdm17yDPzy9ixdY9MZXvl9WFqyfmHnYTkSQfJQWRBtydjTv3U1BUxsKNZby9dGt9E9FNpw3nygm5MV3W+c6K7dz94mK27K7gxlOH8d3zx9KjS8s7R+uuMBrerzuP3TyR3D5NPaNG5MgoKUiHV7b/4CHXkxcWl7NzXzDqdNfO6Uwc0YfrT8njnGMGtrgNeW9lNf85YwWPz1nP4J6Z/MeXTuDzYwfE9N6GVxj9z3Un06urHiYm8aWkIB1O3ZUyM1eUUFBUxrrS4EmgZjBmQI/wSpLe5Odmc9TAHq1yueaCDbu464VFrNq+l/OPG8SoAc0/nmD5lj3845PtXDUhl598qWVXGIkcLiUF6TC2767g6bkbeeaj4Gaq/lldGJ/76SWF43KyD6t5J1aV1TU8PHMNv39vbUxXkXROT+Mb54zhq2ce/hVGIi2lpCDt3sKNu3h89nr+ungLVTXO2WMHcPPk4XxudD9dKSPSQKxJIfnusRZpwsHqWt5YvIXHZq+nsKiMrC6duP7UYdx42nBG9DucJ0uKSCQlBUkJJXsqg/Fjwiaikf27c9+lx3HZSTlxbRoS6Wj03yRJb03JXq763RxK9x5UE5FInCkpSFLbXHaAG/4wF4A3/2VK3EcAFenolBQkaZXureT6P85lT2U1z95+qhKCSBvQBdKSlHZXVHHTox+xuewAj9488bAHARORllFSkJgV7dzPrdPnUVhUFtfPOXCwhv87fT4rtu7h4etPZuLwIxtfSERip6QgMXvmo43BnbjT5vDWkq1x+YyD1bV87ekFzNuwk19dlc/nj45t6AgRaR1KChITd+eNxVs4KS+bsYN6csfTC5j23hpa8+bHmlrn238u5J0VJfy/L53AF08c0mrbFpHYKClITJZu3s2GHfu5ckIuz95+KhceP5j/98Yn3P3yEqprjvyhKO7OD19ZwmuFm7nrgrFcMymvFaIWkZbS1UcSk78u3kJ6mnHecYPI7JzOf18znry+3Xh45hqKdx3goWvHH9ETuR6csYJn5m7kjrNG8dUzR7Vi5CLSEjpTkGbVNR1NHtWXPt2DZ/2mpRnfO38sD1x+ArNXl/Llh+dQvGv/YW3/kXfX8PDMNVx7Sh7/9oWjWzN0EWkhJQVpVl3T0UVRHhN51cQ8Hr91EpvLDzD1odktujJpd0UVf5i1lp+9+QkXjxvM/Zcer1FDRRJMzUfSrMimo2hOH92PF++YzC3T53HVtDn811XjOf/4Q8tW1dSyYuue+ofeFBSVsaZkL+5w1tH9+eWV+W32sHQRaZySgjQpWtNRNGMGZvHS107nK0/M546nF/Cd845mWN9uFGwMEsCSzeVUVAUd0n26Z5Cfm80lJw4hPzebyaP6tsoDb0TkyCkpSJPqmo7uiKHzt39WF569/VS+/XwhP5+xAoAundI4fmgvrp00jPy8bMbnZpPTu6uaiUSSlJKCNKm5pqOG6q5MunJiLn26ZTB2cJYeNymSQpQUpFGxNh01lJZmnHlU/zhGJiLxoq9w0qimrjoSkfZJSUEa1dKmIxFJfUoKEtXhNh2JSGpTUpCo1HQk0jEpKUhUajoS6ZiUFOQz1HQk0nEpKchnqOlIpONSUpDPUNORSMelpCCHUNORSMempCCHUNORSMcW16RgZueb2QozW21md0VZn2dm75jZQjNbZGYXxjMeaZ6ajkQ6trglBTNLBx4CLgCOBa4xs2MbFPt34Hl3Hw9cDfxPvOKR5qnpSETieaYwCVjt7mvd/SDwLHBpgzIO9AynewGb4xiPNENNRyISz6QwFCiKmC8Ol0W6F7jezIqBN4CvR9uQmd1uZvPNbH5JSUk8YhXUdCQi8U0K0Z6i4g3mrwGmu3sOcCHwpJl9JiZ3n+buE9x9Qv/+GpI5HtR0JCIQ36RQDORGzOfw2eah24DnAdx9DpAJ9ItjTNIINR2JCMQ3KcwDxpjZCDPLIOhIfrVBmY3AOQBmdgxBUlD7UAKo6UhEII5Jwd2rgTuBGcBygquMlprZfWZ2SVjs28BXzKwQ+BNws7s3bGKSOFPTkYjUievjON39DYIO5Mhl90RMLwNOj2cM0rSKqhre+WQ7G3bs544zRyU6HBFJMD2juQOprXXWlu6joKiMgqJdFBaVs3zLbqprnexundV0JCJKCu1ZVU0t760sCZNAGYVFZeyuqAage0Y643Ky+coZI8nPzWbCsN5qOhIRJYX27L7XlvHkhxtIMzh6UE8uGjeE8bnZ5OdlM6p/D9LTol01LCIdmZJCO1VRVcPLCzdx8bjBPPjlcXTL0K4WkeZplNR26u/Lt7OnspprJ+UpIYhIzJQU2qmXFm5iUM9MThnZN9GhiEgKUVJoh3buO8jMFdu5JH+I+g1EpEWUFNqhvy7eQnWtMzW/4fiDIiJNU1Joh15euImjB2ZxzOCsRIciIilGSaGd2bhjPws27GLq+KGYqelIRFpGSaGdeblgEwCX5g9JcCQikoqUFNoRd+flgk2cOrIPQ7K7JjocEUlBSgrtyOJN5awt2ceXxquDWUQOj5JCO/LSwk1kdErj/OP1oBwROTxKCu1EdU0trxVu5pyxA+jVtXOiwxGRFKWk0E68v7qU0r0HmaqmIxE5AkoK7cTLCzfRq2tnzjq6f6JDEZEUpqTQDuyrrGbG0m1cNG4wXTqlJzocEUlhSgrtwNvLtnKgqkZXHYnIEVNSaAdeXriZnN5dOTmvd6JDEZEUp6SQ4kr2VDJrVQlT84eSphFRReQIKSmkuNcKN1PrMHW8hrUQkSOnpJDiXi7YxPFDezJ6gEZEFZEjp6SQwlZv38ui4nI9N0FEWo2SQgp7pWATaQaXnKimIxFpHUoKKcrdeWnhJk4f3Y8BPTMTHY6ItBNKCilqwYZdFO86oHsTRKRVKSmkqJcLNtG1czpfOG5QokMRkXak2aRgZgPN7I9m9mY4f6yZ3Rb/0KQxB6treX3RFs47biDdu3RKdDgi0o7EcqYwHZgB1PVmrgS+Ga+ApHnvriyhbH+VRkQVkVYXS1Lo5+7PA7UA7l4N1MQ1KmnSCwuK6ds9gymj+yU6FBFpZ2JJCvvMrC/gAGZ2KlAe16ikUetL9/H2sq1cOTGXTunqEhKR1hVLg/S/Aq8Co8zsA6A/cEVco5JGTZu1lk7padx6+ohEhyIi7VAsSWEpcCZwNGDACnTVUkJs31PBXxYU8+WTc+if1SXR4YhIOxTLwX2Ou1e7+1J3X+LuVcCcWDZuZueb2QozW21mdzVS5kozW2ZmS83smZYE39E89sF6qmtquX3KyESHIiLtVKNnCmY2CBgKdDWz8QRnCQA9gW7NbdjM0oGHgP8DFAPzzOxVd18WUWYM8H3gdHffZWYDDrsm7dzuiiqemrOBC04YzPB+3RMdjoi0U001H30BuBnIAX4ZsXwP8IMYtj0JWO3uawHM7FngUmBZRJmvAA+5+y4Ad98ec+QdzDNzN7Knspo7zhyV6FBEpB1rNCm4++PA42Z2ubu/cBjbHgoURcwXA6c0KHMUQNiBnQ7c6+5vNdyQmd0O3A6Ql5d3GKGktoqqGv74/jqmjOnH8UN7JTocEWnHmu1odvcXzOwi4DggM2L5fc28NdpjwDzK548BziI4I5llZse7e1mDGKYB0wAmTJjQcBvt3ksLN1Gyp5JfX5Wf6FBEpJ2LZZiLR4CrgK8THOivAIbFsO1iIDdiPgfYHKXMK+5e5e7rCK5sGhPDtjuMmlrnd++uYVxOL04b1TfR4YhIOxfL1UeT3f1GYJe7/xg4jUMP9o2ZB4wxsxFmlgFcTXC/Q6SXgc8DmFk/guaktbEG3xHMWLqV9Tv289UzR2GmZzCLSHzFkhQOhD/3m9kQoApo9s6pcDiMOwnGTVoOPO/uS83sPjO7JCw2A9hhZsuAd4DvuvuOllaivXJ3Hp65hhH9ums0VBFpE7HcvPa6mWUDPwc+JugX+EMsG3f3N4A3Giy7J2LaCe6Y/tdYA+5IZq/ZweJN5fz0shNIT9NZgojEXywdzfeHky+Y2etAprtr7KM28PDMNQzI6sJlJ2k0VBFpGy0arsLdK4FJZva3OMUjocXF5by/upRbPzeCLp3SEx2OiHQQjSYFMzvbzFaa2V4zeyp8uM584GfAw20XYsf0yLtryMrsxHWndLz7MkQkcZo6U/gFwQ1jfYG/AB8CT7r7ye7+YlsE11GtK93Hm0u2cP2pw8jK7JzocESkA2mqT8HdfWY4/bKZlbj7r9sgpg5v2nvB8Ni3nD480aGISAfTVFLINrPLIuYtcl5nC/GxfXcFLywo5ssTchiQldn8G0REWlFTSeFd4IuNzDugpBAHj36wnupaDY8tIonR1IB4t7RlIBIMj/30hxoeW0QSR09QSyJPf6jhsUUksZQUksjTczdw+ui+Gh5bRBJGSSFJbN9dQfGuA5w9dmCiQxGRDiyWobOvMLOscPrfzexFMzsp/qF1LAVFwSMk8nN1liAiiRPLmcIP3X2PmX2O4BGdj6M7mltdYXEZ6WnGcUOUFEQkcWJJCjXhz4uAh939FSAjfiF1TIVF5YwdlEVmZ41zJCKJE0tS2GRmvwOuBN4wsy4xvk9iVFvrFBaXkZ+bnehQRKSDi+XgfiXBw3DOD5+d3Af4blyj6mDWlu5jT0U1JyopiEiCxfKQncHAX9290szOAsYBT8Q1qg6msL6TWUlBRBIrljOFF4AaMxsN/JHgUZzPxDWqDqawuIweXToxqn+PRIciIh1cLEmhNnze8mXAf7n7twjOHqSVFBSVccLQXnrkpogkXCxJocrMrgFuBF4Pl2mQ/1ZSUVXD8i271Z8gIkkhlqRwC3Aa8B/uvs7MRgBPxTesjmP5lt1U1bj6E0QkKTTb0ezuy8zse0BeOL+O4JGc0grUySwiySSWYS6+CBQAb4Xz+Wb2arwD6ygKisoY2LMLg3rpgToiknixNB/dC0wCygDcvYDgCiRpBYXF5ZyYo7MEEUkOsSSFancvb7DM4xFMR1O2/yDrSveRn6ekICLJIZab15aY2bVAupmNAb4BzI5vWB1DYXGQa/N1piAiSSKWM4WvA8cBlQQ3rZUD34xnUB1FYVEZZnBCjkZGFZHkEMvVR/uBu8OXtKLCojJG9+9BVqZu+xCR5BDL1Ud/M7PsiPneZjYjvmG1f+5OQVGZbloTkaQSS/NRv3B0VADcfRcwIH4hdQzFuw6wY99BJQURSSoxjX1kZnl1M2Y2DF19dMQKi8Ob1tTJLCJJJJarj+4G3jezd8P5M4Db4xdSx1BYVEZGpzTGDs5KdCgiIvVi6Wh+y8xOAk4FDPiWu5fGPbJ2rqCojOOH9KRzuh5iJyLJI5aO5i8BVe7+uru/BlSb2dT4h9Z+VdfUsnhTufoTRCTpxPI19UeRdzSHnc4/il9I7d/KbXupqKrVIHgiknRiSQrRysTSF4GZnW9mK8xstZnd1US5L5uZm9mEWLab6go0MqqIJKlYksJ8M/ulmY0ys5Fm9itgQXNvMrN04CHgAuBY4BozOzZKuSyCoTPmtiz01FVYVEbvbp3J69Mt0aGIiBwi1mEuDgLPAX8GKoB/juF9k4DV7r7W3Q8CzwKXRil3P/BguN0OobA4uGnNTI/fFJHkEsvVR/uARpt+mjAUKIqYLwZOiSxgZuOBXHd/3cy+09iGzOx2wstg8/LyGiuWEvZVVrNy2x6+cNygRIciIvIZzSYFM3uHKDerufvZzb01yrL67ZhZGvAr4ObmYnD3acA0gAkTJqT0jXOLN5VT6+pPEJHkFEuHceQ3+EzgcqA6hvcVA7kR8znA5oj5LOB4YGbYjDIIeNXMLnH3+TFsPyXVPX5znEZGFZEkFEvzUcNO5Q8i7m5uyjxgjJmNADYBVwPXRmy3HOhXN29mM4HvtOeEAEF/Ql6fbvTt0SXRoYiIfEYszUd9ImbTgJMJvtU3yd2rzexOYAaQDjzq7kvN7D5gvrt3yOc8F2ws4+ThfZovKCKSALE0Hy0g6AswgmajdcBtsWzc3d8A3miw7J5Gyp4VyzZT2fbdFWwur+BWNR2JSJKKpfloRFsE0hHUPX5zvJ7JLCJJqtH7FMxsopkNipi/0cxeMbPfNGhSkhgVFO0iPc04bojOFEQkOTV189rvCG5aw8zOAH4GPEHwjOZp8Q+t/SksKmfsoCwyO6cnOhQRkaiaSgrp7r4znL4KmObuL7j7D4HR8Q+tfamtdQqLy3R/gogktSaTgpnV9TmcA/wjYl1MA+LJp9bt2MeeimoNly0iSa2pg/ufgHfNrBQ4AMwCMLPRBE1I0gIFGzUyqogkv0aTgrv/h5n9HRgMvO3udcNLpBEMkictUFhcRveMdEb175HoUEREGtVkM5C7fxhl2cr4hdN+FRaVMS4nm/Q0jYwqIslLDwhuAxVVNSzbslv9CSKS9JQU2sDyLbupqnHyc3V/gogkNyWFNlBY//jN3gmORESkaUoKbaCgqIyBPbswqFdmokMREWmSkkIbKCwu58Qc9SeISPJTUoiz0r2VrCvdx/g8NR2JSPJTUoiz91eVAvC50f2aKSkiknhKCnH23qoSenfrzHFDeiY6FBGRZikpxJG7M2tVKZ8b05803bQmIilASSGOPtm6h5I9lUwZo6YjEUkNSgpxNGtVCQBnjOmf4EhERGKjpBBHs1aVctTAHro/QURShpJCnFRU1TB33U6m6CxBRFKIkkKczF23k4PVtepPEJGUoqQQJ7NWlpDRKY1TRvRNdCgiIjFTUoiTWatKmTS8D10z0hMdiohIzJQU4mDb7gpWbNujpiMRSTlKCnHZweyEAAAOxElEQVTw3srgUlR1MotIqlFSiINZq0rp16MLxwzOSnQoIiItoqTQymprnfdXl3LGmH6YaWgLEUktSgqtbNmW3ezcd5ApR6k/QURSj5JCK3s37E84XUNli0gKUlJoZbNWlXDM4J4MyNLQFiKSepQUWtG+ymoWbNjFGWo6EpEUpaTQiuau20FVjWtUVBFJWUoKrei9laVkdk7j5GF6HrOIpKa4JgUzO9/MVpjZajO7K8r6fzWzZWa2yMz+bmbD4hlPvL23qoRTRvQls7OGthCR1BS3pGBm6cBDwAXAscA1ZnZsg2ILgQnuPg74C/BgvOKJt+Jd+1lbso8zjlLTkYikrnieKUwCVrv7Wnc/CDwLXBpZwN3fcff94eyHQE4c44mr91eVAnCGxjsSkRQWz6QwFCiKmC8OlzXmNuDNaCvM7HYzm29m80tKSloxxNbz3qoSBvXMZPSAHokORUTksMUzKUQb48GjFjS7HpgA/Dzaenef5u4T3H1C//7J1zxTU+u8v6qUKRraQkRSXKc4brsYyI2YzwE2NyxkZucCdwNnuntlHOOJm0XFZeyuqFZ/goikvHieKcwDxpjZCDPLAK4GXo0sYGbjgd8Bl7j79jjGElezVpVipqEtRCT1xS0puHs1cCcwA1gOPO/uS83sPjO7JCz2c6AH8GczKzCzVxvZXFJ7b2UJJwztRZ/uGYkORUTkiMSz+Qh3fwN4o8GyeyKmz43n57eF3RVVLCwq46tnjkx0KCIiR0x3NB+hOWt2UFOroS1EpH1QUjhCs1aV0D0jnfF5GtpCRFKfksIRem9lKaeN6ktGJ/0qRST16Uh2BDbs2MfGnfuZoqYjEWknlBSOwHt1Q1vo/gQRaSeUFI7ArJUl5PTuyvC+3RIdiohIq1BSOExVNbXMXrODKWP6a2gLEWk3lBQO06xVJeytrOaso9V0JCLth5LCYXp89gYG9uzC2WMHJDoUEZFWo6RwGNaW7OXdlSVcd8owOqfrVygi7YeOaIfhiTkb6JxuXD0pt/nCIiIpREmhhfZWVvPCgmIuOmEwA7IyEx2OiEirUlJooZc+LmZPZTU3TR6e6FBERFqdkkILuDuPz9nAuJxe5OdmJzocEZFWp6TQArPX7GD19r3cdNpw3ZsgIu2SkkILTJ+9nj7dM7ho3OBEhyIiEhdKCjEq2rmfvy/fxjWTcsnsnJ7ocERE4kJJIUZPzd2AmXHdKcMSHYqISNwoKcSgoqqG5+YVcd6xAxmS3TXR4YiIxI2SQgxeLdhM2f4qXYYqIu2ekkIz3J3ps9dz9MAsThnRJ9HhiIjElZJCMxZs2MWyLbu5abIuQxWR9k9JoRnTZ6+nZ2Ynpo4fkuhQRETiTkmhCdt2V/DWkq1cOSGXbhmdEh2OiEjcKSk04em5G6lx54bTdBmqiHQMSgqNOFhdyzNzN/L5owcwrG/3RIcjItImlBQa8eaSLZTurdRlqCLSoSgpNGL67PWM6NedKaP7JToUEZE2o6QQxaLiMhZuLOPG04aRlqbLUEWk41BSiOLx2RvolpHO5SfnJDoUEZE2paTQwJbyA7y2aDOXn5RDz8zOiQ5HRKRN6eL7kLvzwsebuP/1ZRhw8+nDEx2SiEibU1IgeFbCD15azKxVpUwc3pufXjaOUf17JDosEZE216GTQk2t89gH6/jF2ytJTzPun3o8103KU+eyiHRYHTYpLN+ym7teWERhcTnnjB3A/VOP17MSRKTDi2tSMLPzgV8D6cAf3P1nDdZ3AZ4ATgZ2AFe5+/p4xlRRVcNv/7GaR95dQ6+unfnva8Zz8bjBGgFVRIQ4JgUzSwceAv4PUAzMM7NX3X1ZRLHbgF3uPtrMrgYeAK6KV0wfrdvJXS8uYm3JPi47aSg/vOhYenfPiNfHiYiknHieKUwCVrv7WgAzexa4FIhMCpcC94bTfwF+a2bm7t7awfzx/XXc//oycnp35YlbJ3HGUf1b+yNERFJePJPCUKAoYr4YOKWxMu5ebWblQF+gNLKQmd0O3B7O7jWzFYcZU78NUHrmXYf57uTSjwa/pxTWXurSXuoBqkuyOpK6xDTcczyTQrRG+oZnALGUwd2nAdOOOCCz+e4+4Ui3kwxUl+TTXuoBqkuyaou6xPOO5mIgN2I+B9jcWBkz6wT0AnbGMSYREWlCPJPCPGCMmY0wswzgauDVBmVeBW4Kp78M/CMe/QkiIhKbuDUfhX0EdwIzCC5JfdTdl5rZfcB8d38V+CPwpJmtJjhDuDpe8YSOuAkqiaguyae91ANUl2QV97qYvpiLiEgdjZIqIiL1lBRERKReh0kKZna+ma0ws9VmlvR3KpjZejNbbGYFZjY/XNbHzP5mZqvCn73D5WZmvwnrtsjMTkpw7I+a2XYzWxKxrMWxm9lNYflVZnZTtM9KUF3uNbNN4b4pMLMLI9Z9P6zLCjP7QsTyhP79mVmumb1jZsvNbKmZ/Uu4POX2SxN1ScX9kmlmH5lZYViXH4fLR5jZ3PB3/Fx4sQ5m1iWcXx2uH95cHVvM3dv9i6Cjew0wEsgACoFjEx1XMzGvB/o1WPYgcFc4fRfwQDh9IfAmwX0fpwJzExz7GcBJwJLDjR3oA6wNf/YOp3snSV3uBb4Tpeyx4d9WF2BE+DeXngx/f8Bg4KRwOgtYGcabcvulibqk4n4xoEc43RmYG/6+nweuDpc/AtwRTn8NeCScvhp4rqk6Hk5MHeVMoX7IDXc/CNQNuZFqLgUeD6cfB6ZGLH/CAx8C2WY2OBEBArj7e3z2fpOWxv4F4G/uvtPddwF/A86Pf/SHaqQujbkUeNbdK919HbCa4G8v4X9/7r7F3T8Op/cAywlGFEi5/dJEXRqTzPvF3X1vONs5fDlwNsHQP/DZ/VK3v/4CnGNmRuN1bLGOkhSiDbnR1B9RMnDgbTNbYMEwHwAD3X0LBP8YwIBweSrUr6WxJ3ud7gybVR6ta3IhReoSNjmMJ/hWmtL7pUFdIAX3i5mlm1kBsJ0gya4Byty9OkpchwwNBNQNDdRqdekoSSGm4TSSzOnufhJwAfDPZnZGE2VTsX51Gos9mev0MDAKyAe2AL8Ilyd9XcysB/AC8E13391U0SjLkr0uKblf3L3G3fMJRn2YBBwTrVj4M+516ShJIZYhN5KKu28Of24HXiL4Y9lW1ywU/tweFk+F+rU09qStk7tvC/+Ra4Hf8+lpelLXxcw6ExxEn3b3F8PFKblfotUlVfdLHXcvA2YS9ClkWzD0T8O4GhsaqNXq0lGSQixDbiQNM+tuZll108B5wBIOHRbkJuCVcPpV4MbwipFTgfK6JoEk0tLYZwDnmVnvsBngvHBZwjXor/kSwb6BoC5Xh1eIjADGAB+RBH9/YbvzH4Hl7v7LiFUpt18aq0uK7pf+ZpYdTncFziXoI3mHYOgf+Ox+iTY0UGN1bLm27GlP5IvgaoqVBO11dyc6nmZiHUlwJUEhsLQuXoK2w78Dq8KfffzTKxgeCuu2GJiQ4Pj/RHD6XkXwDea2w4kduJWgw2w1cEsS1eXJMNZF4T/j4Ijyd4d1WQFckCx/f8DnCJoTFgEF4evCVNwvTdQlFffLOGBhGPMS4J5w+UiCg/pq4M9Al3B5Zji/Olw/srk6tvSlYS5ERKReR2k+EhGRGCgpiIhIPSUFERGpp6QgIiL1lBRERKSekoIkBTNzM/tFxPx3zOzeVtr2dDP7cvMlj/hzrghH7nynwfLhZnYgYvTOAjO7sZlt3Wdm57ZCTHubLyXyqbg9jlOkhSqBy8zsp+5emuhg6phZurvXxFj8NuBr7v5OlHVrPBjKICbufk+sZUVak84UJFlUEzx/9lsNVzT8pl/37dfMzjKzd83seTNbaWY/M7PrwvHpF5vZqIjNnGtms8JyF4fvTzezn5vZvHAQtX+K2O47ZvYMwc1QDeO5Jtz+EjN7IFx2D8FNVY+Y2c9jrbSZ7TWzX5jZx2b2dzPr37DOYb2WhTH+Z7hsWFh+UfgzL1w+wszmhHW6v8FnfTeirnXj9nc3s79aMJ7/EjO7KtbYpX1SUpBk8hBwnZn1asF7TgT+BTgBuAE4yt0nAX8Avh5RbjhwJnARwYE7k+Cbfbm7TwQmAl8JhwiAYNycu9392MgPM7MhwAMEQxvnAxPNbKq73wfMB65z9+9GiXNUg+ajKeHy7sDHHgx++C7wowaf14dgyIbj3H0c8JNw1W8JhrYeBzwN/CZc/mvg4bBOWyO2cx7B0AeTwrhPtmCQxfOBze5+orsfD7wVJXbpQJQUJGl4MNLlE8A3WvC2eR6Mr19JcIv/2+HyxQSJoM7z7l7r7qsIHgwzlmDcnhstGLZ4LsGQD2PC8h95MC59QxOBme5e4sHQxU8TPIinOWvcPT/iNStcXgs8F04/RXC2EWk3UAH8wcwuA/aHy08Dngmnn4x43+kEQ3PULa9zXvhaCHwc1n8Mwe/pXDN7wMymuHt5DHWRdkx9CpJs/ovgoPVYxLJqwi8w4WBoGRHrKiOmayPmazn077vheC51ww1/3d0PGdDNzM4C9jUSX7QhilvTIXG6e7WZTQLOIRiw7U6Cs5Sm3hdt7BoDfuruv/vMCrOTCcYA+qmZvR2e9UgHpTMFSSruvpPgUYS3RSxeD5wcTl9K8HSqlrrCzNLCfoaRBIOGzQDusGAYZszsKAtGpW3KXOBMM+tnZunANQTNPocrjU9Hw7wWeD9ypQXPDOjl7m8A3yRo+gGYTZAkAK6LeN8HDZbXmQHcGm4PMxtqZgPC5rD97v4U8J8Ejx6VDkxnCpKMfkHwjbjO74FXzOwjgpE8G/sW35QVBAfvgcBX3b3CzP5A0MT0cXgGUsKnjz2Myt23mNn3CYY2NuANd3+lqfeERoXNVHUedfffENTlODNbQPAUrYYdvVkEdc8MP6+uI/4bwKNm9t0w7lvC5f8CPGPBw+xfiIj7bTM7BpgTVJW9wPXAaODnZlZLMBLsHTHURdoxjZIqkkBmttfdeyQ6DpE6aj4SEZF6OlMQEZF6OlMQEZF6SgoiIlJPSUFEROopKYiISD0lBRERqff/AdMQhuc6EPjcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(1)\n",
    "plt.plot(episodes, success_rate)\n",
    "plt.title('HER performance on N={} bits'.format(NUM_BITS))\n",
    "plt.ylabel('Success Rate')\n",
    "plt.xlabel('Number of Episodes')\n",
    "plt.ylim([0, 1])\n",
    "if SAVE_MODEL:\n",
    "    plt.savefig(str(NUM_BITS)+FIGURE_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save the model parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Weights saved\n",
      "Model saved\n",
      "Complete\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/afs/inf.ed.ac.uk/user/s17/s1791387/miniconda3/envs/her/lib/python3.6/site-packages/torch/serialization.py:193: UserWarning: Couldn't retrieve source code for container of type FNN. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
     ]
    }
   ],
   "source": [
    "if SAVE_MODEL:\n",
    "    torch.save(policy_net.state_dict(), str(NUM_BITS)+WEIGHTS_PATH)\n",
    "    print('Weights saved')\n",
    "    torch.save(policy_net, str(NUM_BITS)+MODEL_PATH)\n",
    "    print('Model saved')\n",
    "\n",
    "print('Complete')  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the model parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "LOAD_MODEL = False\n",
    "if LOAD_MODEL:\n",
    "    policy_net.load_state_dict(torch.load('final_weights/'+str(NUM_BITS)+WEIGHTS_PATH))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99\n"
     ]
    }
   ],
   "source": [
    "successes = 0\n",
    "for i in range(100):\n",
    "    env.reset()\n",
    "    test_state = env.init_state\n",
    "    goal_state = env.target_state\n",
    "#     print('#############################################')\n",
    "#     print('start', test_state)\n",
    "#     print('goal', goal_state)\n",
    "    next_state = test_state.clone()\n",
    "    for i in range(NUM_BITS):\n",
    "        action = select_action(next_state, goal_state, greedy=True)\n",
    "        next_state, reward = env.step(action.item())\n",
    "#         print('taking action', action)\n",
    "#         print('next state', next_state)\n",
    "        if np.array_equal(next_state, goal_state):\n",
    "            successes += 1\n",
    "            break\n",
    "print(successes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}