2c_Breakout-v0-recurrent_Test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7d1fa20c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys\n",
    "import gc\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "759b713c",
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.append('../')\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c8c767df",
   "metadata": {},
   "outputs": [],
   "source": [
    "from get_algos import get_all_algos , run_experiment ,call_paths, plot_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c9fc3531",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "from sb3_contrib import QRDQN ,RecurrentPPO, TRPO\n",
    "from stable_baselines3.common.vec_env import VecFrameStack,VecVideoRecorder\n",
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "from stable_baselines3.common.env_util import make_atari_env\n",
    "from stable_baselines3.common.monitor import Monitor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ed55ba4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "env_name='Breakout-v0'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54c5cdd5",
   "metadata": {},
   "source": [
    "### 1. Test\n",
    "<ol>\n",
    "    <li>Create the environment</li>\n",
    "    <li>For each algorithm:</li>\n",
    "    <ol><li>Load the model</li>\n",
    "        <li>Evaluate the model for 5 sample iterations </li>\n",
    "        <li>Output the score  for each algo and each iteration</li>\n",
    "        <li>Render the video and store it </li>\n",
    "    </ol>\n",
    "</ol> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "65fae03b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['QRDQN', 'RecurrentPPO', 'TRPO']\n"
     ]
    }
   ],
   "source": [
    "env_recurrent_type={'Discrete':['MaskablePPO', 'QRDQN', 'RecurrentPPO', 'TRPO'],'Box':['ARS', 'RecurrentPPO', 'TQC', 'TRPO'], \n",
    "                    'MultiDiscrete':['MaskablePPO', 'RecurrentPPO', 'TRPO'],'MultiBinary':['MaskablePPO', 'RecurrentPPO', 'TRPO']}\n",
    "def get_algo_recurrent(env):\n",
    "    act_space=str(type(env.action_space)).split('.')[3].split(\"'\")[0]\n",
    "    return env_recurrent_type[act_space]\n",
    "\n",
    "algo_list=get_algo_recurrent(gym.make(env_name))\n",
    "algo_list.remove('MaskablePPO') if 'MaskablePPO' in algo_list else algo_list\n",
    "print(algo_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e8fab60",
   "metadata": {},
   "source": [
    "### 3. Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "972110da",
   "metadata": {},
   "outputs": [],
   "source": [
    "def testing_model(algo_list, env_name,n_steps,vid_length):\n",
    "    total_rewards={}\n",
    "    for algo_name in reversed(algo_list):\n",
    "        algo_rewards=[]\n",
    "        if algo_name =='QRDQN' or 'TRPO':\n",
    "            device='cpu'\n",
    "        else:\n",
    "            device='cuda' #GPU was unable to load the entire experience relay so moved to virtual memory on disk\n",
    "        policy='CnnLstmPolicy' if algo_name=='RecurrentPPO' else 'CnnPolicy'\n",
    "        log_path, render_path, model_path=call_paths(algo_name,env_name,n_steps)\n",
    "        env=VecVideoRecorder(VecFrameStack(make_atari_env(env_name,monitor_dir=render_path),n_stack=6),render_path,record_video_trigger=lambda step: step == 0,video_length=vid_length)\n",
    "        model_name=env_name+\"_\"+algo_name+\"_model\"\n",
    "        model=eval(algo_name).load(os.path.join(model_path, model_name),env,device=device)\n",
    "        for i in range(5):\n",
    "            state=env.reset()\n",
    "            epi_rewards=0\n",
    "            while True:\n",
    "                action,_=model.predict(state)\n",
    "                state,reward,done,_=env.step(action)\n",
    "                epi_rewards+=reward[0]\n",
    "                env.render()\n",
    "                if done:\n",
    "                    algo_rewards.append(epi_rewards)\n",
    "                    break\n",
    "        total_rewards[algo_name]=algo_rewards\n",
    "        del model\n",
    "        env.close()\n",
    "    return total_rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2f2d2d60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wrapping the env in a VecTransposeImage.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\ai_gym\\lib\\site-packages\\gym\\envs\\atari\\environment.py:269: UserWarning: \u001b[33mWARN: We strongly suggest supplying `render_mode` when constructing your environment, e.g., gym.make(ID, render_mode='human'). Using `render_mode` provides access to proper scaling, audio support, and proper framerates.\u001b[0m\n",
      "  \"We strongly suggest supplying `render_mode` when \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wrapping the env in a VecTransposeImage.\n",
      "Wrapping the env in a VecTransposeImage.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\ai_gym\\lib\\site-packages\\stable_baselines3\\common\\buffers.py:229: UserWarning: This system does not have apparently enough memory to store the complete replay buffer 84.69GB > 10.47GB\n",
      "  \"This system does not have apparently enough memory to store the complete \"\n"
     ]
    }
   ],
   "source": [
    "test=total_rewards=testing_model(algo_list, env_name,n_steps=200000,vid_length=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "63f6b45f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'TRPO': [0.0, 0.0, 0.0, 0.0, 0.0],\n",
       " 'RecurrentPPO': [0.0, 5.0, 0.0, 0.0, 0.0],\n",
       " 'QRDQN': [0.0, 0.0, 2.0, 1.0, 0.0]}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "870561de",
   "metadata": {},
   "source": [
    "We see that RecurrentPPO seems to consistently provide rewards with the highest average reward. Hence, we will use RecurrentPPO as the algorithm for subsequent training of <b> 5,000,000 </b>steps"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}