RL_flappy_bird.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8d2cf13e",
   "metadata": {},
   "source": [
    "### Deep Learning Final Project\n",
    "#### Playing Modified Flappy Bird Using Reinforcement Learning\n",
    "Leo Li(zl3493), Zhangnan Jiang(zj2028), Zichen Yang(zy2486) <br>\n",
    "In this project, we would like to build our own reinforcement learning neural network to play the\n",
    "Flappy Bird game. <br>\n",
    "Please find all supporting files in the same folder. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36bad027",
   "metadata": {},
   "source": [
    "#### Importing necessary libraries:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fea20bbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import cv2\n",
    "import shutil\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchsummary\n",
    "from random import random, randint, sample\n",
    "from tensorboardX import SummaryWriter"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f405c5aa",
   "metadata": {},
   "source": [
    "#### Defining function used to preprocess game frame data (converting to grayscale):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "16710bb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def pre_processing(image, width, height):\n",
    "    \n",
    "    image = cv2.cvtColor(cv2.resize(image, (width, height)), cv2.COLOR_BGR2GRAY)\n",
    "    _, image = cv2.threshold(image, 1, 255, cv2.THRESH_BINARY)\n",
    "    return image[None, :, :].astype(np.float32)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84cff25a",
   "metadata": {},
   "source": [
    "#### Defining Model Structure:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adb5c38a",
   "metadata": {},
   "source": [
    "Deep Q Network:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "25bd2340",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DeepQNetwork(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(DeepQNetwork, self).__init__()\n",
    "        self.conv1 = nn.Sequential(nn.Conv2d(4, 32, kernel_size=8, stride=4), nn.ReLU(inplace=True))\n",
    "        self.conv2 = nn.Sequential(nn.Conv2d(32, 64, kernel_size=4, stride=2), nn.ReLU(inplace=True))\n",
    "        self.conv3 = nn.Sequential(nn.Conv2d(64, 64, kernel_size=3, stride=1), nn.ReLU(inplace=True))\n",
    "\n",
    "        self.fc1 = nn.Sequential(nn.Linear(7 * 7 * 64, 512), nn.ReLU(inplace=True))\n",
    "        self.fc2 = nn.Linear(512, 2)\n",
    "        \n",
    "        self.init_weights()\n",
    "        \n",
    "    # Initiate model weights\n",
    "    def init_weights(self):\n",
    "        for module in self.modules():\n",
    "            if isinstance(module, nn.Conv2d) or isinstance(module, nn.Linear):\n",
    "                nn.init.uniform_(module.weight, -0.01, 0.01)\n",
    "                nn.init.constant_(module.bias, 0)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = self.conv2(x)\n",
    "        x = self.conv3(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.fc1(x)\n",
    "        x = self.fc2(x)\n",
    "\n",
    "        return x\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1cbaf6fe",
   "metadata": {},
   "source": [
    "Dueling DQN:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ab0a0aef",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DDQN(nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.conv1 = nn.Conv2d(4, 32, kernel_size=8, stride=4)\n",
    "        self.bn1 = nn.BatchNorm2d(32)\n",
    "        self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)\n",
    "        self.bn2 = nn.BatchNorm2d(64)\n",
    "        self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)\n",
    "        self.bn3 = nn.BatchNorm2d(64)\n",
    "\n",
    "        linear_input_size = 7 * 7 * 64\n",
    "        fc_output_size=512\n",
    "        self.fc_val=nn.Linear(linear_input_size, fc_output_size)\n",
    "        self.fc_adv=nn.Linear(linear_input_size, fc_output_size)\n",
    "        self.val = nn.Linear(fc_output_size, 1)\n",
    "        self.adv = nn.Linear(fc_output_size, 2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.float()\n",
    "        x = F.relu(self.bn1(self.conv1(x)))\n",
    "        x = F.relu(self.bn2(self.conv2(x)))\n",
    "        x = F.relu(self.bn3(self.conv3(x)))\n",
    "        \n",
    "        x_val = F.relu(self.fc_val(x.view(x.size(0), -1)))\n",
    "        x_adv = F.relu(self.fc_adv(x.view(x.size(0), -1)))\n",
    "        val=self.val(x_val)\n",
    "        adv=self.adv(x_adv)\n",
    "        \n",
    "        x=val+adv-adv.mean(1,keepdim=True)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c8299906",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7fa1a2d",
   "metadata": {},
   "source": [
    "#### The training procedure:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c475f0cc",
   "metadata": {},
   "source": [
    "Defining hyperparameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6bf4b999",
   "metadata": {},
   "outputs": [],
   "source": [
    "saved_path = \"results\"\n",
    "log_path = \"ddqnlog\"\n",
    "image_size = 84\n",
    "lr = 1e-6\n",
    "num_iters = 2000000\n",
    "initial_epsilon = 0.1\n",
    "final_epsilon = 1e-4\n",
    "memory_buffer_size = 46000\n",
    "gamma = 0.99\n",
    "batch_size = 32"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c2f929a",
   "metadata": {},
   "source": [
    "Load model select (can choose: Deep Q Network/ Dueling DQN, pretrain or not):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a4d98f94",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.manual_seed(123)\n",
    "model = DDQN()\n",
    "# model = torch.load(\"results/flappy_bird\")\n",
    "# torchsummary.summary(model, (4, 8, 8))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "73fd0ad4",
   "metadata": {},
   "source": [
    "Handling tensorboard log path:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fa86a920",
   "metadata": {},
   "outputs": [],
   "source": [
    "if os.path.isdir(log_path):\n",
    "        shutil.rmtree(log_path)\n",
    "os.makedirs(log_path)\n",
    "writer = SummaryWriter(log_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "749721e4",
   "metadata": {},
   "source": [
    "We can monitor the training progress by checking the embedded tensorboard: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7539fa83",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "      <iframe id=\"tensorboard-frame-d7b2bb52348fc69e\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
       "      </iframe>\n",
       "      <script>\n",
       "        (function() {\n",
       "          const frame = document.getElementById(\"tensorboard-frame-d7b2bb52348fc69e\");\n",
       "          const url = new URL(\"/\", window.location);\n",
       "          url.port = 6006;\n",
       "          frame.src = url;\n",
       "        })();\n",
       "      </script>\n",
       "  "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%load_ext tensorboard\n",
    "%tensorboard --logdir ddqnlog"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e7be398",
   "metadata": {},
   "source": [
    "Using Adam Optimizer and MSE loss between y_batch and q_value:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5a7ba608",
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "criterion = nn.MSELoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b357aeb9",
   "metadata": {},
   "source": [
    "Starting the game and take the first frame as the initial input:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ea1c44f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pygame 2.1.2 (SDL 2.0.16, Python 3.6.15)\n",
      "Hello from the pygame community. https://www.pygame.org/contribute.html\n"
     ]
    }
   ],
   "source": [
    "from src.flappy_bird import FlappyBird\n",
    "game_state = FlappyBird() # start the game\n",
    "image, reward, terminal = game_state.next_frame(0) # get the frame from the started game\n",
    "image = pre_processing(image[:game_state.screen_width, :int(game_state.base_y)], image_size, image_size)\n",
    "image = torch.from_numpy(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "87704f53",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.cuda()\n",
    "image = image.cuda()\n",
    "state = torch.cat(tuple(image for _ in range(4)))[None, :, :, :]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bab57ed7",
   "metadata": {},
   "source": [
    "Initialize the relay buffer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "07a351a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "memory_buffer = []\n",
    "iter = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "396a0681",
   "metadata": {},
   "source": [
    "The training iterations:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9862d6fd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    while iter < num_iters:\n",
    "        \n",
    "        prediction = model(state)[0]\n",
    "\n",
    "        epsilon = final_epsilon + ((num_iters - iter) * (initial_epsilon - final_epsilon) / num_iters)\n",
    "        iter += 1\n",
    "        \n",
    "        if random() <= epsilon:\n",
    "            action = randint(0, 1)# make a random game action\n",
    "        else:\n",
    "            action = torch.argmax(prediction)\n",
    "        \n",
    "        # get the next frame from the started game applying a new game action\n",
    "        next_image, reward, terminal = game_state.next_frame(action) \n",
    "        next_image = pre_processing(next_image[:game_state.screen_width, :int(game_state.base_y)], image_size, image_size) # preprocess the game frame\n",
    "        next_image = torch.from_numpy(next_image)\n",
    "        next_image = next_image.cuda()\n",
    "        next_state = torch.cat((state[0, 1:, :, :], next_image))[None, :, :, :]\n",
    "        \n",
    "        memory_buffer.append([state, action, reward, next_state, terminal])\n",
    "        \n",
    "        if len(memory_buffer) > memory_buffer_size:\n",
    "            del memory_buffer[0]\n",
    "        \n",
    "        batch = sample(memory_buffer, min(len(memory_buffer), batch_size))\n",
    "        \n",
    "        state_batch, action_batch, reward_batch, next_state_batch, terminal_batch = zip(*batch)\n",
    "\n",
    "        state_batch = torch.cat(tuple(state for state in state_batch))\n",
    "        action_batch = torch.from_numpy(np.array([[1, 0] if action == 0 else [0, 1] for action in action_batch], dtype=np.float32))\n",
    "        reward_batch = torch.from_numpy(np.array(reward_batch, dtype=np.float32)[:, None])\n",
    "        next_state_batch = torch.cat(tuple(state for state in next_state_batch))\n",
    "\n",
    "        state_batch = state_batch.cuda()\n",
    "        action_batch = action_batch.cuda()\n",
    "        reward_batch = reward_batch.cuda()\n",
    "        next_state_batch = next_state_batch.cuda()\n",
    "        \n",
    "        current_prediction_batch = model(state_batch)\n",
    "        next_prediction_batch = model(next_state_batch)\n",
    "\n",
    "        y_batch = torch.cat(tuple(reward if terminal else reward + gamma * torch.max(prediction) for reward, terminal, prediction in\n",
    "zip(reward_batch, terminal_batch, next_prediction_batch)))\n",
    "\n",
    "        q_value = torch.sum(current_prediction_batch * action_batch, dim=1)\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        loss = criterion(q_value, y_batch) # MSE\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        state = next_state\n",
    "        \n",
    "        if iter % 1000 == 0:\n",
    "            print(\"Iteration: {}/{}, Action: {}, Loss: {}, Epsilon {}, Reward: {}, Q-value: {}\".format(\n",
    "                iter + 1,\n",
    "                num_iters,\n",
    "                action,\n",
    "                loss,\n",
    "                epsilon, reward, torch.max(prediction)))\n",
    "            writer.add_scalar('Train/Loss', loss, iter)\n",
    "            writer.add_scalar('Train/Epsilon', epsilon, iter)\n",
    "            writer.add_scalar('Train/Reward', reward, iter)\n",
    "            writer.add_scalar('Train/Q-value', torch.max(prediction), iter)\n",
    "\n",
    "        if (iter+1) % 1000000 == 0:\n",
    "            torch.save(model.state_dict(), \"{}/fireball_flappy_bird_rewarded_ddqn{}.pth\".format(saved_path, iter+1))\n",
    "except KeyboardInterrupt:\n",
    "    print(\"Saving model before quit\")\n",
    "    torch.save(model.state_dict(), \"{}/fireball_flappy_bird_rewarded_ddqn{}.pth\".format(saved_path, iter+1))\n",
    "\n",
    "torch.save(model.state_dict(), \"{}/fireball_flappy_bird_rewarded_ddqn.pth\".format(saved_path))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3beac9b4",
   "metadata": {},
   "source": [
    "#### Test Procedure:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "549cba12",
   "metadata": {},
   "outputs": [],
   "source": [
    "saved_path = \"results\"\n",
    "image_size = 84\n",
    "testing_rounds = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "24573d4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.manual_seed(123)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "41efcee5",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = DeepQNetwork()\n",
    "# model = DDQN()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d36cc199",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from src.deep_q_network import DeepQNetwork\n",
    "from src.flappy_bird import FlappyBird\n",
    "import copy\n",
    "\n",
    "def test(model_name, testing_rounds):\n",
    "    if torch.cuda.is_available():\n",
    "        model.load_state_dict(torch.load(\"{}/{}\".format(saved_path, model_name)))\n",
    "    else:\n",
    "        model.load_state_dict(torch.load(\"{}/{}\".format(saved_path, model_name), map_location=torch.device('cpu')))\n",
    "\n",
    "    model.eval()\n",
    "\n",
    "    scores = []\n",
    "\n",
    "    try:\n",
    "        game_state = FlappyBird()\n",
    "        image, reward, terminal = game_state.next_frame(0)\n",
    "        image = pre_processing(image[:game_state.screen_width, :int(game_state.base_y)], image_size, image_size)\n",
    "        image = torch.from_numpy(image)\n",
    "        if torch.cuda.is_available():\n",
    "            model.cuda()\n",
    "            image = image.cuda()\n",
    "        state = torch.cat(tuple(image for _ in range(4)))[None, :, :, :]\n",
    "\n",
    "        while True:\n",
    "            if testing_rounds == 0:\n",
    "                break\n",
    "            prediction = model(state)[0]\n",
    "            action = torch.argmax(prediction)\n",
    "\n",
    "            next_image, reward, terminal = game_state.next_frame(action)\n",
    "            if terminal:\n",
    "                testing_rounds = testing_rounds - 1\n",
    "            next_image = pre_processing(next_image[:game_state.screen_width, :int(game_state.base_y)], image_size,\n",
    "                                        image_size)\n",
    "            next_image = torch.from_numpy(next_image)\n",
    "            if torch.cuda.is_available():\n",
    "                next_image = next_image.cuda()\n",
    "            next_state = torch.cat((state[0, 1:, :, :], next_image))[None, :, :, :]\n",
    "\n",
    "            state = next_state\n",
    "    except KeyboardInterrupt:\n",
    "        print(\"Quit\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62cd9fcb",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "test(\"dqn_rewarded_flappy_bird_2000000.pth\",100)\n",
    "# Score output below is saved in a txt file named score.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25fbbeb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_ddqn(model_name, testing_rounds):\n",
    "    if torch.cuda.is_available():\n",
    "        model = torch.load(\"{}/{}\".format(saved_path, model_name))\n",
    "    else:\n",
    "        model = torch.load(\"{}/{}\".format(saved_path, model_name), map_location=torch.device('cpu'))\n",
    "\n",
    "    model.eval()\n",
    "\n",
    "    scores = []\n",
    "\n",
    "    try:\n",
    "        game_state = FlappyBird()\n",
    "        image, reward, terminal = game_state.next_frame(0)\n",
    "        image = pre_processing(image[:game_state.screen_width, :int(game_state.base_y)], image_size, image_size)\n",
    "        image = torch.from_numpy(image)\n",
    "        if torch.cuda.is_available():\n",
    "            model.cuda()\n",
    "            image = image.cuda()\n",
    "        state = torch.cat(tuple(image for _ in range(4)))[None, :, :, :]\n",
    "\n",
    "        while True:\n",
    "            if testing_rounds == 0:\n",
    "                break\n",
    "            prediction = model(state)[0]\n",
    "            action = torch.argmax(prediction)\n",
    "\n",
    "            next_image, reward, terminal = game_state.next_frame(action)\n",
    "            if terminal:\n",
    "                testing_rounds = testing_rounds - 1\n",
    "            next_image = pre_processing(next_image[:game_state.screen_width, :int(game_state.base_y)], image_size,\n",
    "                                        image_size)\n",
    "            next_image = torch.from_numpy(next_image)\n",
    "            if torch.cuda.is_available():\n",
    "                next_image = next_image.cuda()\n",
    "            next_state = torch.cat((state[0, 1:, :, :], next_image))[None, :, :, :]\n",
    "\n",
    "            state = next_state\n",
    "    except KeyboardInterrupt:\n",
    "        print(\"Quit\")\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "342e031f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from src.ddqn4 import DDQN4\n",
    "test_ddqn(\"fireball_flappy_bird_rewarded_ddqn2000000.pth\",100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "de360860",
   "metadata": {},
   "outputs": [],
   "source": [
    "file1 = open('score/fireball_flappy_bird_rewarded_ddqn1000000.txt', 'r')\n",
    "Lines = file1.readlines()\n",
    "  \n",
    "count = 0\n",
    "score = []\n",
    "# Strips the newline character\n",
    "for line in Lines:\n",
    "    count += 1\n",
    "    score.append(int(line.strip()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a18e6a64",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max: 8\n",
      "Average: 3.19\n"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "print(\"Max: \" + str(numpy.max(score)))\n",
    "print(\"Average: \" + str(numpy.average(score)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}