cgames/01_ping_pong/ping_pong_rainbow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LN0nZwyMGadB"
   },
   "source": [
    "# Pong with Dueling Dqn\n",
    "\n",
    "## Step 1: Import the libraries"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import gym\n",
    "import random\n",
    "import torch\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.display import clear_output\n",
    "import math\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../../')\n",
    "from algos.agents import DDQNAgent\n",
    "from algos.models import DDQNCnn\n",
    "from algos.preprocessing.stack_frame import preprocess_frame, stack_frame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "tfo8jleHGadK"
   },
   "source": [
    "## Step 2: Create our environment\n",
    "\n",
    "Initialize the environment in the code cell below.\n"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('Pong-v0')\n",
    "env.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if gpu is to be used\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"Device: \", device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nS221MgXGadP"
   },
   "source": [
    "## Step 3: Viewing our Enviroment"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"The size of frame is: \", env.observation_space.shape)\n",
    "print(\"No. of Actions: \", env.action_space.n)\n",
    "env.reset()\n",
    "plt.figure()\n",
    "plt.imshow(env.reset())\n",
    "plt.title('Original Frame')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Execute the code cell below to play Pong with a random policy."
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_play():\n",
    "    score = 0\n",
    "    env.reset()\n",
    "    while True:\n",
    "        env.render()\n",
    "        action = env.action_space.sample()\n",
    "        state, reward, done, _ = env.step(action)\n",
    "        score += reward\n",
    "        if done:\n",
    "            env.close()\n",
    "            print(\"Your Score at end of game is: \", score)\n",
    "            break\n",
    "random_play()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Sr52nmcpGada"
   },
   "source": [
    "## Step 4:Preprocessing Frame"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.reset()\n",
    "plt.figure()\n",
    "plt.imshow(preprocess_frame(env.reset(), (30, -4, -12, 4), 84), cmap=\"gray\")\n",
    "plt.title('Pre Processed image')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mJMc3HA8Gade"
   },
   "source": [
    "## Step 5: Stacking Frame"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stack_frames(frames, state, is_new=False):\n",
    "    frame = preprocess_frame(state, (30, -4, -12, 4), 84)\n",
    "    frames = stack_frame(frames, frame, is_new)\n",
    "\n",
    "    return frames\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6: Creating our Agent"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_SHAPE = (4, 84, 84)\n",
    "ACTION_SIZE = env.action_space.n\n",
    "SEED = 0\n",
    "GAMMA = 0.99           # discount factor\n",
    "BUFFER_SIZE = 100000   # replay buffer size\n",
    "BATCH_SIZE = 1024      # Update batch size\n",
    "LR = 0.002             # learning rate \n",
    "TAU = .1               # for soft update of target parameters\n",
    "UPDATE_EVERY = 100     # how often to update the network\n",
    "UPDATE_TARGET = 10000  # After which thershold replay to be started \n",
    "EPS_START = 0.99       # starting value of epsilon\n",
    "EPS_END = 0.01         # Ending value of epsilon\n",
    "EPS_DECAY = 100        # Rate by which epsilon to be decayed\n",
    "\n",
    "agent = DDQNAgent(INPUT_SHAPE, ACTION_SIZE, SEED, device, BUFFER_SIZE, BATCH_SIZE, GAMMA, LR, TAU, UPDATE_EVERY, UPDATE_TARGET, DDQNCnn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 7: Watching untrained agent play"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# watch an untrained agent\n",
    "state = stack_frames(None, env.reset(), True) \n",
    "for j in range(200):\n",
    "    env.render()\n",
    "    action = agent.act(state, .9)\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    state = stack_frames(state, next_state, False)\n",
    "    if done:\n",
    "        break \n",
    "        \n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 8: Loading Agent\n",
    "Uncomment line to load a pretrained agent"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_epoch = 0\n",
    "scores = []\n",
    "scores_window = deque(maxlen=20)\n",
    "\n",
    "# To Load checkpoint uncomment code\n",
    "# checkpoint = torch.load('space_invader_dqn.pth')\n",
    "# agent.policy_net.load_state_dict(checkpoint['state_dict'])\n",
    "# agent.target_net.load_state_dict(checkpoint['state_dict'])\n",
    "# agent.optimizer.load_state_dict(checkpoint['optimizer'])\n",
    "# start_epoch = checkpoint['epoch']\n",
    "# scores = checkpoint['socres']\n",
    "\n",
    "# index = 1\n",
    "# for i in reversed(scores):\n",
    "#     scores_window.append(i)\n",
    "#     if index == 100:\n",
    "#         break\n",
    "#     index += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 9: Train the Agent with DQN"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon_by_epsiode = lambda frame_idx: EPS_END + (EPS_START - EPS_END) * math.exp(-1. * frame_idx /EPS_DECAY)\n",
    "\n",
    "plt.plot([epsilon_by_epsiode(i) for i in range(1000)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(n_episodes=1000):\n",
    "    \"\"\"\n",
    "    Params\n",
    "    ======\n",
    "        n_episodes (int): maximum number of training episodes\n",
    "    \"\"\"\n",
    "    for i_episode in range(start_epoch + 1, n_episodes+1):\n",
    "        state = stack_frames(None, env.reset(), True)\n",
    "        score = 0\n",
    "        eps = epsilon_by_epsiode(i_episode)\n",
    "        while True:\n",
    "            action = agent.act(state, eps)\n",
    "            next_state, reward, done, info = env.step(action)\n",
    "            score += reward\n",
    "            next_state = stack_frames(state, next_state, False)\n",
    "            agent.step(state, action, reward, next_state, done)\n",
    "            state = next_state\n",
    "            if done:\n",
    "                break\n",
    "        scores_window.append(score)       # save most recent score\n",
    "        scores.append(score)              # save most recent score\n",
    "        \n",
    "        clear_output(True)\n",
    "        fig = plt.figure()\n",
    "        ax = fig.add_subplot(111)\n",
    "        plt.plot(np.arange(len(scores)), scores)\n",
    "        plt.ylabel('Score')\n",
    "        plt.xlabel('Episode #')\n",
    "        plt.show()\n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}\\tEpsilon: {:.2f}'.format(i_episode, np.mean(scores_window), eps), end=\"\")\n",
    "    \n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = train(1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(len(scores)), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 10: Watch a Smart Agent!"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = 0\n",
    "state = stack_frames(None, env.reset(), True)\n",
    "while True:\n",
    "    env.render()\n",
    "    action = agent.act(state, .01)\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    score += reward\n",
    "    state = stack_frames(state, next_state, False)\n",
    "    if done:\n",
    "        print(\"You Final score is:\", score)\n",
    "        break \n",
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "space_invader.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3.7.6 64-bit ('ai': conda)",
   "language": "python",
   "name": "python37664bitaicondac0e8880bb07b42c396c674eb5a2d8b93"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}