SAC_Value.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Target\n",
    "1. From OpenAI\n",
    "    1. At test time, to see how well the policy exploits what it has learned, remove stochasticity and use the mean action instead of a sample from the distribution. This tends to improve performance over the original stochastic policy.\n",
    "    1. Explore randomly prior to start SAC befor n_steps\n",
    "    1. Use Value Network\n",
    "    1. Set proper entropy based on formula from OpenAI H = - log(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import the necessary library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "from unityagents import UnityEnvironment\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:unityagents:\n",
      "'Academy' started successfully!\n",
      "Unity Academy name: Academy\n",
      "        Number of Brains: 1\n",
      "        Number of External Brains : 1\n",
      "        Lesson number : 0\n",
      "        Reset Parameters :\n",
      "\t\t\n",
      "Unity brain name: CrawlerBrain\n",
      "        Number of Visual Observations (per agent): 0\n",
      "        Vector Observation space type: continuous\n",
      "        Vector Observation space size (per agent): 129\n",
      "        Number of stacked Vector Observation: 1\n",
      "        Vector Action space type: continuous\n",
      "        Vector Action space size (per agent): 20\n",
      "        Vector Action descriptions: , , , , , , , , , , , , , , , , , , , \n"
     ]
    }
   ],
   "source": [
    "env = UnityEnvironment(file_name=\"./Crawler_Windows_x86_64/Crawler.exe\")\n",
    "# env = UnityEnvironment(file_name=\"./Reacher_Windows_x86_64/Reacher.exe\")\n",
    "# get the default brain\n",
    "brain_name = env.brain_names[0]\n",
    "brain = env.brains[brain_name]\n",
    "\n",
    "env_info = env.reset(train_mode=False)[brain_name]\n",
    "num_agents = len(env_info.agents)\n",
    "states = env_info.vector_observations\n",
    "action_size = brain.vector_action_space_size\n",
    "state_size = states.shape[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Method to plot the progress of the agent's score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_result(scores):\n",
    "    # plot the scores\n",
    "    fig = plt.figure()\n",
    "    ax = fig.add_subplot(111)\n",
    "    plt.plot(np.arange(len(scores)), scores)\n",
    "    plt.ylabel('Score')\n",
    "    plt.xlabel('Episode #')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(agent, scores=[], n_episodes=500, train_mode=True, episode_start=1, start_at = 1000):\n",
    "    scores_window = deque(maxlen=100)  # last 100 scores\n",
    "    for s in scores[-100:]:\n",
    "        scores_window.append(s)\n",
    "    frame_no = 0\n",
    "    for i_episode in range(episode_start, n_episodes+1):\n",
    "        env_info = env.reset(train_mode=train_mode)[brain_name]\n",
    "        states = env_info.vector_observations\n",
    "        scores_one_episode = np.zeros(num_agents)\n",
    "        while True:\n",
    "            frame_no += 1\n",
    "            if(frame_no < start_at):\n",
    "                actions = np.random.randn(num_agents, action_size) # select a random action (for each agent)\n",
    "            else:\n",
    "                actions = agent.act(states)              # select an action (for each agent)\n",
    "                \n",
    "            env_info = env.step(np.clip(actions, -1, 1))[brain_name]              # send all actions to the environment\n",
    "            next_states = env_info.vector_observations                            # get next state (for each agent)\n",
    "            rewards = env_info.rewards                                            # get reward (for each agent)\n",
    "            dones = env_info.local_done                                           # see if episode finished\n",
    "            agent.step(states, actions, rewards, next_states, dones)              # learn\n",
    "            states = next_states                                                  # roll over states to next time step\n",
    "            \n",
    "            scores_one_episode += rewards\n",
    "            if np.any(dones):                                                     # exit loop if episode finished\n",
    "                break\n",
    "                       \n",
    "        score = np.average(scores_one_episode)\n",
    "        scores.append(score)\n",
    "        scores_window.append(score)\n",
    "        mean_100 = np.mean(scores_window)\n",
    "        \n",
    "        if i_episode % 50 == 0:\n",
    "            print('\\rEpisode {}\\tAvg: {:.3f}\\tMin: {:.3f}\\tMax: {:.3f}\\talpha: {:.3f}\\tPLoss: {:.3f}\\tCLoss: {:.3f}\\tEst: {:.3f}'.\n",
    "                  format(i_episode, mean_100, \n",
    "                             np.min(np.array(scores_window)[-50:]),\n",
    "                             np.max(np.array(scores_window)[-50:]),\n",
    "                             agent.network.log_alpha.exp().cpu().detach().numpy().item(),\n",
    "                             agent.policy_loss, np.mean(agent.critics_losses),\n",
    "                             agent.estimation))\n",
    "            Storage.save(\"weights\\SAC_Value\\eps_{}_avg_{:.3f}.pth\".format(i_episode, mean_100), scores, agent=agent)\n",
    "            \n",
    "        if len(scores_window) >= 100 and np.mean(scores_window)>=2000:\n",
    "            print('\\nEnvironment solved in {:d} episodes!\\tAverage Score: {:.3f}'.format(i_episode, mean_100))\n",
    "            Storage.save(\"weights\\SAC_Value\\final.pth\", scores, agent=agent)\n",
    "            break\n",
    "            \n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import agents_maddpg\n",
    "import random\n",
    "\n",
    "# device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "device = \"cpu\"\n",
    "def set_seed(seed):\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    np.random.seed(seed)\n",
    "    random.seed(seed)\n",
    "\n",
    "set_seed(354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Every step, there are 12 new experiences. So before any update, there are 32x12 new experiences. If, MADDPG is sample efficient by 4 times, therefore, every update, 32x12 experiences should be learned 4 times!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from agents_maddpg.storage_sac import Storage\n",
    "from agents_maddpg.model import TanhGaussianActorCriticValue\n",
    "import torch.nn.functional as F\n",
    "agent = Storage.new_sac_value( TanhGaussianActorCriticValue, states.shape[1], action_size, device,  \n",
    "                    memory_size=int(1e5),\n",
    "                    batch_size=32,\n",
    "                    ACTIVATION = F.leaky_relu,\n",
    "                    TAU=1e-2,\n",
    "                    LR_CRITIC = 1e-4,\n",
    "                    LR_ACTOR = 1e-4,\n",
    "                    LR_ALPHA = 1e-4,\n",
    "                    UPDATE_EVERY=1,\n",
    "                    TRANSFER_EVERY=1,\n",
    "                    UPDATE_LOOP=1,\n",
    "                    GAMMA=0.99,\n",
    "                    TARGET_ENTROPY=1.2,\n",
    "                    Q_NUMBER = 3,\n",
    "                    WEIGHT_DECAY = 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test save and load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from agents_maddpg.sac_value import SAC_Value\n",
    "Storage.save(\"temp.ckp\", [], agent)\n",
    "loaded, scores = Storage.load(\"temp.ckp\", device, agent_class = SAC_Value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 50\tAvg: 0.565\tMin: -0.120\tMax: 3.021\talpha: 0.946\tPLoss: -49.631\tCLoss: 48.030\tEst: 36.290\n",
      "Episode 100\tAvg: 0.503\tMin: -9.767\tMax: 12.849\talpha: 0.854\tPLoss: -137.085\tCLoss: 7.346\tEst: 124.477\n",
      "Episode 150\tAvg: 0.966\tMin: -8.776\tMax: 11.872\talpha: 0.652\tPLoss: -293.127\tCLoss: 5.822\tEst: 284.478\n",
      "Episode 200\tAvg: 1.735\tMin: -4.921\tMax: 15.248\talpha: 0.490\tPLoss: -358.814\tCLoss: 10.067\tEst: 352.681\n",
      "Episode 250\tAvg: 2.904\tMin: -4.654\tMax: 20.957\talpha: 0.372\tPLoss: -372.314\tCLoss: 7.204\tEst: 367.173\n",
      "Episode 300\tAvg: 3.183\tMin: -6.475\tMax: 13.052\talpha: 0.282\tPLoss: -374.410\tCLoss: 3.291\tEst: 370.747\n",
      "Episode 350\tAvg: 2.112\tMin: -7.204\tMax: 13.285\talpha: 0.202\tPLoss: -321.441\tCLoss: 2.247\tEst: 318.748\n",
      "Episode 400\tAvg: 1.507\tMin: -10.376\tMax: 17.065\talpha: 0.150\tPLoss: -280.228\tCLoss: 1.754\tEst: 278.406\n",
      "Episode 450\tAvg: 1.391\tMin: -9.667\tMax: 14.051\talpha: 0.112\tPLoss: -244.433\tCLoss: 1.207\tEst: 243.252\n",
      "Episode 500\tAvg: 0.453\tMin: -20.580\tMax: 11.557\talpha: 0.089\tPLoss: -208.950\tCLoss: 0.667\tEst: 207.907\n",
      "Episode 550\tAvg: 0.519\tMin: -14.387\tMax: 23.214\talpha: 0.063\tPLoss: -146.958\tCLoss: 0.267\tEst: 146.169\n",
      "Episode 600\tAvg: 0.073\tMin: -13.778\tMax: 9.560\talpha: 0.048\tPLoss: -127.871\tCLoss: 495.878\tEst: 127.353\n",
      "Episode 650\tAvg: 0.202\tMin: -8.252\tMax: 10.376\talpha: 0.036\tPLoss: -105.848\tCLoss: 0.262\tEst: 105.395\n",
      "Episode 700\tAvg: 0.887\tMin: -7.472\tMax: 12.677\talpha: 0.027\tPLoss: -70.830\tCLoss: 0.205\tEst: 70.550\n",
      "Episode 750\tAvg: -1.495\tMin: -15.062\tMax: 7.866\talpha: 0.022\tPLoss: -56.453\tCLoss: 16.928\tEst: 56.266\n",
      "Episode 800\tAvg: -2.829\tMin: -18.889\tMax: 8.523\talpha: 0.017\tPLoss: -55.246\tCLoss: 0.267\tEst: 55.107\n",
      "Episode 850\tAvg: -3.310\tMin: -12.638\tMax: 4.356\talpha: 0.013\tPLoss: -28.896\tCLoss: 0.057\tEst: 28.817\n",
      "Episode 900\tAvg: -4.503\tMin: -14.936\tMax: 1.508\talpha: 0.010\tPLoss: -18.899\tCLoss: 0.033\tEst: 18.833\n",
      "Episode 950\tAvg: -4.585\tMin: -11.889\tMax: 5.153\talpha: 0.007\tPLoss: -7.117\tCLoss: 0.029\tEst: 7.108\n",
      "Episode 1000\tAvg: -4.325\tMin: -17.379\tMax: 0.763\talpha: 0.006\tPLoss: -6.868\tCLoss: 0.045\tEst: 6.852\n"
     ]
    }
   ],
   "source": [
    "# loaded.network.log_alpha = torch.nn.Parameter(torch.tensor(-0.5, dtype=torch.float32))\n",
    "scores = train(loaded, scores, n_episodes=60000,train_mode=True)\n",
    "plot_result(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "loaded, scores = Storage.load(\"weights/SAC_Value/eps_4250_avg_53.859.pth\", device)\n",
    "scores = train(loaded, scores, n_episodes=60000, train_mode=True, episode_start=len(scores)+1, start_at=0)\n",
    "plot_result(scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### View the trained agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loaded, scores = Storage.load(\"weights/SAC_Value/eps_3700_avg_1.200.pth\", device)\n",
    "loaded.network.eval()\n",
    "env_info = env.reset(train_mode=False)[brain_name]\n",
    "\n",
    "def act(network, states, device):\n",
    "    states = torch.from_numpy(states).float().unsqueeze(0).to(device)\n",
    "    ret = network(states).squeeze().cpu().data.numpy()\n",
    "    return ret\n",
    "\n",
    "for i in range(20):\n",
    "    print(i)\n",
    "    while True:\n",
    "        actions = act(loaded.network, states, device)\n",
    "        env_info = env.step(np.clip(actions, -1, 1))[brain_name]  # send all actions to the environment\n",
    "        states = env_info.vector_observations                     # get next state (for each agent)\n",
    "        dones = env_info.local_done                               # see if episode finished\n",
    "        if np.any(dones):                                         # exit loop if episode finished\n",
    "            break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "drlnd",
   "language": "python",
   "name": "drlnd"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}