cgames/05_sonic/sonic_ddqn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "colab_type": "text",
    "id": "LN0nZwyMGadB"
   },
   "outputs": [],
   "source": [
    "# Sonic The Hedgehog 1 with Dueling dqn\n",
    "\n",
    "## Step 1: Import the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import retro\n",
    "import random\n",
    "import torch\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.display import clear_output\n",
    "import math\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../../')\n",
    "from algos.agents.dqn_agent import DDQNAgent\n",
    "from algos.models.dqn_cnn import DDQNCnn\n",
    "from algos.preprocessing.stack_frame import preprocess_frame, stack_frame"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "colab_type": "text",
    "id": "tfo8jleHGadK"
   },
   "outputs": [],
   "source": [
    "## Step 2: Create our environment\n",
    "\n",
    "Initialize the environment in the code cell below.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = retro.make(game='SonicTheHedgehog-Genesis', state='GreenHillZone.Act1', scenario='contest')\n",
    "env.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if gpu is to be used\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"Device: \", device)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "colab_type": "text",
    "id": "nS221MgXGadP"
   },
   "outputs": [],
   "source": [
    "## Step 3: Viewing our Enviroment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"The size of frame is: \", env.observation_space.shape)\n",
    "print(\"No. of Actions: \", env.action_space.n)\n",
    "env.reset()\n",
    "plt.figure()\n",
    "plt.imshow(env.reset())\n",
    "plt.title('Original Frame')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "possible_actions = {\n",
    "            # No Operation\n",
    "            0: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "            # Left\n",
    "            1: [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0],\n",
    "            # Right\n",
    "            2: [0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0],\n",
    "            # Left, Down\n",
    "            3: [0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0],\n",
    "            # Right, Down\n",
    "            4: [0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0],\n",
    "            # Down\n",
    "            5: [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0],\n",
    "            # Down, B\n",
    "            6: [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0],\n",
    "            # B\n",
    "            7: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
    "        }"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Execute the code cell below to play Pong with a random policy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_play():\n",
    "    score = 0\n",
    "    env.reset()\n",
    "    for i in range(200):\n",
    "        env.render()\n",
    "        action = possible_actions[np.random.randint(len(possible_actions))]\n",
    "        state, reward, done, _ = env.step(action)\n",
    "        score += reward\n",
    "        if done:\n",
    "            print(\"Your Score at end of game is: \", score)\n",
    "            break\n",
    "    env.reset()\n",
    "    env.render(close=True)\n",
    "random_play()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "colab_type": "text",
    "id": "Sr52nmcpGada"
   },
   "outputs": [],
   "source": [
    "## Step 4:Preprocessing Frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "plt.imshow(preprocess_frame(env.reset(), (1, -1, -1, 1), 84), cmap=\"gray\")\n",
    "plt.title('Pre Processed image')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "colab_type": "text",
    "id": "mJMc3HA8Gade"
   },
   "outputs": [],
   "source": [
    "## Step 5: Stacking Frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stack_frames(frames, state, is_new=False):\n",
    "    frame = preprocess_frame(state, (1, -1, -1, 1), 84)\n",
    "    frames = stack_frame(frames, frame, is_new)\n",
    "\n",
    "    return frames\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 6: Creating our Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_SHAPE = (4, 84, 84)\n",
    "ACTION_SIZE = len(possible_actions)\n",
    "SEED = 0\n",
    "GAMMA = 0.99           # discount factor\n",
    "BUFFER_SIZE = 100000   # replay buffer size\n",
    "BATCH_SIZE = 32        # Update batch size\n",
    "LR = 0.0001            # learning rate \n",
    "TAU = 1e-3             # for soft update of target parameters\n",
    "UPDATE_EVERY = 100     # how often to update the network\n",
    "UPDATE_TARGET = 10000  # After which thershold replay to be started \n",
    "EPS_START = 0.99       # starting value of epsilon\n",
    "EPS_END = 0.01         # Ending value of epsilon\n",
    "EPS_DECAY = 100         # Rate by which epsilon to be decayed\n",
    "\n",
    "agent = DDQNAgent(INPUT_SHAPE, ACTION_SIZE, SEED, device, BUFFER_SIZE, BATCH_SIZE, GAMMA, LR, TAU, UPDATE_EVERY, UPDATE_TARGET, DDQNCnn)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 7: Watching untrained agent play"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.viewer = None\n",
    "# watch an untrained agent\n",
    "state = stack_frames(None, env.reset(), True) \n",
    "for j in range(200):\n",
    "    env.render(close=False)\n",
    "    action = agent.act(state, eps=0.01)\n",
    "    next_state, reward, done, _ = env.step(possible_actions[action])\n",
    "    state = stack_frames(state, next_state, False)\n",
    "    if done:\n",
    "        env.reset()\n",
    "        break \n",
    "env.render(close=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 8: Loading Agent\n",
    "Uncomment line to load a pretrained agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_epoch = 0\n",
    "scores = []\n",
    "scores_window = deque(maxlen=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 9: Train the Agent with DDQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon_by_epsiode = lambda frame_idx: EPS_END + (EPS_START - EPS_END) * math.exp(-1. * frame_idx /EPS_DECAY)\n",
    "\n",
    "plt.plot([epsilon_by_epsiode(i) for i in range(1000)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(n_episodes=1000):\n",
    "    \"\"\"\n",
    "    Params\n",
    "    ======\n",
    "        n_episodes (int): maximum number of training episodes\n",
    "    \"\"\"\n",
    "    for i_episode in range(start_epoch + 1, n_episodes+1):\n",
    "        state = stack_frames(None, env.reset(), True)\n",
    "        score = 0\n",
    "        eps = epsilon_by_epsiode(i_episode)\n",
    "\n",
    "        # Punish the agent for not moving forward\n",
    "        prev_state = {}\n",
    "        steps_stuck = 0\n",
    "        timestamp = 0\n",
    "\n",
    "        while timestamp < 10000:\n",
    "            action = agent.act(state, eps)\n",
    "            next_state, reward, done, info = env.step(possible_actions[action])\n",
    "            score += reward\n",
    "\n",
    "            timestamp += 1\n",
    "\n",
    "            # Punish the agent for standing still for too long.\n",
    "            if (prev_state == info):\n",
    "                steps_stuck += 1\n",
    "            else:\n",
    "                steps_stuck = 0\n",
    "            prev_state = info\n",
    "    \n",
    "            if (steps_stuck > 20):\n",
    "                reward -= 1\n",
    "            \n",
    "            next_state = stack_frames(state, next_state, False)\n",
    "            agent.step(state, action, reward, next_state, done)\n",
    "            state = next_state\n",
    "            if done:\n",
    "                break\n",
    "        scores_window.append(score)       # save most recent score\n",
    "        scores.append(score)              # save most recent score\n",
    "        \n",
    "        \n",
    "        clear_output(True)\n",
    "        fig = plt.figure()\n",
    "        ax = fig.add_subplot(111)\n",
    "        plt.plot(np.arange(len(scores)), scores)\n",
    "        plt.ylabel('Score')\n",
    "        plt.xlabel('Episode #')\n",
    "        plt.show()\n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}\\tEpsilon: {:.2f}'.format(i_episode, np.mean(scores_window), eps), end=\"\")\n",
    "    \n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = train(1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 10: Watch a Smart Agent!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.viewer = None\n",
    "# watch an untrained agent\n",
    "state = stack_frames(None, env.reset(), True) \n",
    "for j in range(10000):\n",
    "    env.render(close=False)\n",
    "    action = agent.act(state, eps=0.91)\n",
    "    next_state, reward, done, _ = env.step(possible_actions[action])\n",
    "    state = stack_frames(state, next_state, False)\n",
    "    if done:\n",
    "        env.reset()\n",
    "        break \n",
    "env.render(close=True)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "space_invader.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3.7.6 64-bit ('ai': conda)",
   "language": "python",
   "name": "python37664bitaicondac0e8880bb07b42c396c674eb5a2d8b93"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}