cgames/02_space_invader/space_invader_dqn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "colab_type": "text",
    "id": "LN0nZwyMGadB"
   },
   "outputs": [],
   "source": [
    "# Pong with dqn\n",
    "\n",
    "## Step 1: Import the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import gym\n",
    "import random\n",
    "import torch\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../../')\n",
    "from algos.agents.dqn_agent import DQNAgent\n",
    "from algos.models.dqn_cnn import DQNCnn\n",
    "from algos.preprocessing.stack_frame import preprocess_frame, stack_frame"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "colab_type": "text",
    "id": "tfo8jleHGadK"
   },
   "outputs": [],
   "source": [
    "## Step 2: Create our environment\n",
    "\n",
    "Initialize the environment in the code cell below.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('SpaceInvaders-v0')\n",
    "env.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if gpu is to be used\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"Device: \", device)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "colab_type": "text",
    "id": "nS221MgXGadP"
   },
   "outputs": [],
   "source": [
    "## Step 3: Viewing our Enviroment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"The size of frame is: \", env.observation_space.shape)\n",
    "print(\"No. of Actions: \", env.action_space.n)\n",
    "env.reset()\n",
    "plt.figure()\n",
    "plt.imshow(env.reset())\n",
    "plt.title('Original Frame')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Execute the code cell below to play Pong with a random policy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_play():\n",
    "    score = 0\n",
    "    env.reset()\n",
    "    while True:\n",
    "        env.render()\n",
    "        action = env.action_space.sample()\n",
    "        state, reward, done, _ = env.step(action)\n",
    "        score += reward\n",
    "        if done:\n",
    "            env.close()\n",
    "            print(\"Your Score at end of game is: \", score)\n",
    "            break\n",
    "random_play()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "colab_type": "text",
    "id": "Sr52nmcpGada"
   },
   "outputs": [],
   "source": [
    "## Step 4:Preprocessing Frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.reset()\n",
    "plt.figure()\n",
    "plt.imshow(preprocess_frame(env.reset(), (8, -12, -12, 4), 84), cmap=\"gray\")\n",
    "plt.title('Pre Processed image')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "colab_type": "text",
    "id": "mJMc3HA8Gade"
   },
   "outputs": [],
   "source": [
    "## Step 5: Stacking Frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stack_frames(frames, state, is_new=False):\n",
    "    frame = preprocess_frame(state, (8, -12, -12, 4), 84)\n",
    "    frames = stack_frame(frames, frame, is_new)\n",
    "\n",
    "    return frames\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 6: Creating our Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_SHAPE = (4, 84, 84)\n",
    "ACTION_SIZE = env.action_space.n\n",
    "SEED = 0\n",
    "GAMMA = 0.99           # discount factor\n",
    "BUFFER_SIZE = 100000   # replay buffer size\n",
    "BATCH_SIZE = 64        # Update batch size\n",
    "LR = 0.0001            # learning rate \n",
    "TAU = 1e-3             # for soft update of target parameters\n",
    "UPDATE_EVERY = 1       # how often to update the network\n",
    "UPDATE_TARGET = 10000  # After which thershold replay to be started \n",
    "EPS_START = 0.99       # starting value of epsilon\n",
    "EPS_END = 0.01         # Ending value of epsilon\n",
    "EPS_DECAY = 100         # Rate by which epsilon to be decayed\n",
    "\n",
    "agent = DQNAgent(INPUT_SHAPE, ACTION_SIZE, SEED, device, BUFFER_SIZE, BATCH_SIZE, GAMMA, LR, TAU, UPDATE_EVERY, UPDATE_TARGET, DQNCnn)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 7: Watching untrained agent play"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# watch an untrained agent\n",
    "state = stack_frames(None, env.reset(), True) \n",
    "for j in range(200):\n",
    "    env.render()\n",
    "    action = agent.act(state)\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    state = stack_frames(state, next_state, False)\n",
    "    if done:\n",
    "        break \n",
    "        \n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 8: Loading Agent\n",
    "Uncomment line to load a pretrained agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_epoch = 0\n",
    "scores = []\n",
    "scores_window = deque(maxlen=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 9: Train the Agent with DQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon_by_epsiode = lambda frame_idx: EPS_END + (EPS_START - EPS_END) * math.exp(-1. * frame_idx /EPS_DECAY)\n",
    "\n",
    "plt.plot([epsilon_by_epsiode(i) for i in range(1000)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(n_episodes=1000):\n",
    "    \"\"\"\n",
    "    Params\n",
    "    ======\n",
    "        n_episodes (int): maximum number of training episodes\n",
    "    \"\"\"\n",
    "    for i_episode in range(start_epoch + 1, n_episodes+1):\n",
    "        state = stack_frames(None, env.reset(), True)\n",
    "        score = 0\n",
    "        eps = epsilon_by_epsiode(i_episode)\n",
    "        while True:\n",
    "            action = agent.act(state, eps)\n",
    "            next_state, reward, done, info = env.step(action)\n",
    "            score += reward\n",
    "            next_state = stack_frames(state, next_state, False)\n",
    "            agent.step(state, action, reward, next_state, done)\n",
    "            state = next_state\n",
    "            if done:\n",
    "                break\n",
    "        scores_window.append(score)       # save most recent score\n",
    "        scores.append(score)              # save most recent score\n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)), end=\"\")\n",
    "        \n",
    "        if i_episode % 100 == 0:\n",
    "            print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))\n",
    "            fig = plt.figure()\n",
    "            ax = fig.add_subplot(111)\n",
    "            plt.plot(np.arange(len(scores)), scores)\n",
    "            plt.ylabel('Score')\n",
    "            plt.xlabel('Episode #')\n",
    "            plt.show()\n",
    "    \n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = train(1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 10: Watch a Smart Agent!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = 0\n",
    "state = stack_frames(None, env.reset(), True)\n",
    "while True:\n",
    "    env.render()\n",
    "    action = agent.act(state)\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    score += reward\n",
    "    state = stack_frames(state, next_state, False)\n",
    "    if done:\n",
    "        print(\"You Final score is:\", score)\n",
    "        break \n",
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "space_invader.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3.7.6 64-bit ('ai': conda)",
   "language": "python",
   "name": "python37664bitaicondac0e8880bb07b42c396c674eb5a2d8b93"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}