notebooks/gfootball-stable-baselines3-test-agent.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import base64\n",
    "import pickle\n",
    "import zlib\n",
    "import gym\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch as th\n",
    "from torch import nn, tensor\n",
    "from collections import deque\n",
    "from gym.spaces import Box, Discrete\n",
    "from gfootball.env import create_environment, observation_preprocessing, wrappers\n",
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.ppo import CnnPolicy\n",
    "from stable_baselines3.common import results_plotter\n",
    "from stable_baselines3.common.callbacks import BaseCallback\n",
    "from stable_baselines3.common.callbacks import EvalCallback\n",
    "from stable_baselines3.common.env_checker import check_env\n",
    "from stable_baselines3.common.monitor import Monitor\n",
    "from stable_baselines3.common.torch_layers import BaseFeaturesExtractor\n",
    "from stable_baselines3.common.vec_env.dummy_vec_env import DummyVecEnv\n",
    "from stable_baselines3.common.vec_env.subproc_vec_env import SubprocVecEnv\n",
    "from stable_baselines3.common.vec_env.base_vec_env import VecEnv\n",
    "from stable_baselines3.common.policies import BasePolicy, register_policy\n",
    "from IPython.display import HTML\n",
    "import time\n",
    "from datetime import date\n",
    "from matplotlib import pyplot as plt\n",
    "from stable_baselines3 import DQN, PPO\n",
    "import torch\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7fee79c1db70>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(42)\n",
    "torch.manual_seed(torch.initial_seed())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Football Gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "class FootballGym(gym.Env):\n",
    "    spec = None\n",
    "    metadata = None\n",
    "#     metadata = {'render.modes': ['human']}\n",
    "    def __init__(self, config=None, render=False, rewards='scoring'):\n",
    "        super(FootballGym, self).__init__()\n",
    "        env_name = \"academy_empty_goal_close\"\n",
    "        rewards = rewards\n",
    "        if config is not None:\n",
    "            env_name = config.get(\"env_name\", env_name)\n",
    "            rewards = config.get(\"rewards\", rewards)\n",
    "        self.env = create_environment(\n",
    "            env_name=env_name,\n",
    "            stacked=False,\n",
    "            representation=\"simple115v2\",\n",
    "            rewards = rewards,\n",
    "            write_goal_dumps=False,\n",
    "            write_full_episode_dumps=False,\n",
    "            render=render,\n",
    "            write_video=False,\n",
    "            dump_frequency=1,\n",
    "            logdir=\".\",\n",
    "            extra_players=None,\n",
    "            number_of_left_players_agent_controls=1,\n",
    "            number_of_right_players_agent_controls=0)\n",
    "        self.action_space = self.env.action_space\n",
    "        self.observation_space = self.env.observation_space\n",
    "        self.reward_range = (-1, 1)\n",
    "        self.obs_stack = deque([], maxlen=4)\n",
    "        \n",
    "    def reset(self):\n",
    "        self.obs_stack.clear()\n",
    "        obs = self.env.reset()\n",
    "        return obs\n",
    "    \n",
    "    def step(self, action):\n",
    "        obs, reward, done, info = self.env.step([action])\n",
    "        return obs, float(reward), done, info\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DDQN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.nn import functional as F\n",
    "\n",
    "class DDQN(DQN):\n",
    "    def train(self, gradient_steps: int, batch_size: int = 100) -> None:\n",
    "        # Switch to train mode (this affects batch norm / dropout)\n",
    "        self.policy.set_training_mode(True)\n",
    "        # Update learning rate according to schedule\n",
    "        self._update_learning_rate(self.policy.optimizer)\n",
    "\n",
    "        losses = []\n",
    "        for _ in range(gradient_steps):\n",
    "            ### YOUR CODE HERE\n",
    "            # Sample replay buffer\n",
    "            replay_data = self.replay_buffer.sample(batch_size, env=self._vec_normalize_env)\n",
    "\n",
    "            # Do not backpropagate gradient to the target network\n",
    "            with th.no_grad():\n",
    "#                 print(f\"replay data actions: {replay_data.actions.shape} and data: {replay_data.actions}\")\n",
    "                # Compute the next Q-values using the target network\n",
    "                next_q_values = self.q_net_target(replay_data.next_observations)\n",
    "#                 print(f\"Next Q values shape calculated by target net: {next_q_values.shape}\")\n",
    "                # Decouple action selection from value estimation\n",
    "                # Compute q-values for the next observation using the online q net\n",
    "                next_q_values_online = self.q_net(replay_data.next_observations)\n",
    "#                 print(f\"Next Q values shape calculated by Online net: {next_q_values_online.shape}\")\n",
    "                # Select action with online network\n",
    "                next_actions_online = next_q_values_online.argmax(dim=1)\n",
    "#                 print(f\"Next Actions shape calculated by Online net: {next_actions_online.shape}\")\n",
    "                # Estimate the q-values for the selected actions using target q network\n",
    "                next_q_values = th.gather(next_q_values, dim=1, index=next_actions_online.unsqueeze(-1))\n",
    "#                 print(f\"Next Q values calculated by Target net from the selected actions: {next_q_values.shape}\")\n",
    "               \n",
    "                # 1-step TD target\n",
    "                target_q_values = replay_data.rewards + (1 - replay_data.dones) * self.gamma * next_q_values\n",
    "\n",
    "            # Get current Q-values estimates\n",
    "            current_q_values = self.q_net(replay_data.observations)\n",
    "\n",
    "            # Retrieve the q-values for the actions from the replay buffer\n",
    "            current_q_values = th.gather(current_q_values, dim=1, index=replay_data.actions.long())\n",
    "\n",
    "            # Check the shape\n",
    "            assert current_q_values.shape == target_q_values.shape\n",
    "\n",
    "            # Compute loss (L2 or Huber loss)\n",
    "            loss = F.smooth_l1_loss(current_q_values, target_q_values)\n",
    "\n",
    "            ### END OF YOUR CODE\n",
    "            \n",
    "            losses.append(loss.item())\n",
    "\n",
    "            # Optimize the q-network\n",
    "            self.policy.optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            # Clip gradient norm\n",
    "            th.nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm)\n",
    "            self.policy.optimizer.step()\n",
    "\n",
    "        # Increase update counter\n",
    "        self._n_updates += gradient_steps\n",
    "\n",
    "        self.logger.record(\"train/n_updates\", self._n_updates, exclude=\"tensorboard\")\n",
    "        self.logger.record(\"train/loss\", np.mean(losses))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tetsing the Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Put the path of the model here\n",
    "model_path = \"../models/ddqn/ddqn_gfootball_8_20-05-2022-23-37-05.zip\"\n",
    "\n",
    "\n",
    "# Just comment the model that you don't want to use\n",
    "model = DDQN.load(model_path)\n",
    "model = PPO.load(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scenarios = {0: \"academy_empty_goal_close\",\n",
    "             1: \"academy_empty_goal\",\n",
    "             2: \"academy_run_to_score\",\n",
    "             3: \"academy_run_to_score_with_keeper\",\n",
    "             4: \"academy_pass_and_shoot_with_keeper\",\n",
    "             5: \"academy_run_pass_and_shoot_with_keeper\",\n",
    "             6: \"academy_3_vs_1_with_keeper\",\n",
    "             7: \"academy_corner\",\n",
    "             8: \"academy_counterattack_easy\",\n",
    "             9: \"academy_counterattack_hard\",\n",
    "             10: \"academy_single_goal_versus_lazy\",\n",
    "             11: \"11_vs_11_kaggle\",\n",
    "             12: \"11_vs_11_stochastic\",\n",
    "             13: \"11_vs_11_easy_stochastic\",\n",
    "             14: \"11_vs_11_hard_stochastic\"}\n",
    "\n",
    "scenario_name = scenarios[13]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "action_set = {\n",
    "    0: \"idle\",\n",
    "    1: \"left\",\n",
    "    2: \"top_left\",\n",
    "    3: \"top\",\n",
    "    4: \"top_right\",\n",
    "    5: \"right\",\n",
    "    6: \"bottom_right\",\n",
    "    7: \"bottom\",\n",
    "    8: \"bottom_left\",\n",
    "    9: \"long_pass\",\n",
    "    10: \"high_pass\",\n",
    "    11: \"short_pass\",\n",
    "    12: \"shot\",\n",
    "    13: \"sprint\",\n",
    "    14: \"release_direction\",\n",
    "    15: \"release_sprint\",\n",
    "    16: \"sliding\",\n",
    "    17: \"dribble\",\n",
    "    18: \"release_dribble\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def play_match_with_dqn_agent(test_env):\n",
    "    obs = test_env.reset()\n",
    "    env_steps = 0\n",
    "    match_reward = 0\n",
    "    done = False\n",
    "    my_agent = 0\n",
    "    ai_agent = 0\n",
    "    while not done:\n",
    "        action, state = model.predict(obs, deterministic=True)\n",
    "        obs, reward, done, info = test_env.step(action)\n",
    "        if reward > 0:\n",
    "            my_agent += 1\n",
    "        elif reward < 0:\n",
    "            ai_agent += 1\n",
    "        env_steps += 1\n",
    "        if (env_steps+1) % 3001  == 0:\n",
    "            return my_agent, ai_agent\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scenario_names = [scenarios[11], scenarios[12], scenarios[13], scenarios[14]]\n",
    "scenario_names = [scenarios[13]]\n",
    "total_matches = 10\n",
    "with torch.no_grad():\n",
    "    for scenario_name in scenario_names:\n",
    "        test_env = FootballGym({\"env_name\":scenario_name}, render=False)\n",
    "        win_count = 0\n",
    "        draw_count = 0\n",
    "        total_goals_scored = 0\n",
    "        total_goal_conceded = 0\n",
    "        print(f\"-------------------------------------------------\")\n",
    "        print(f\"Playing in the {scenario_name} scenario:\")\n",
    "        print(f\"-------------------------------------------------\")\n",
    "        \n",
    "        for i in range(total_matches):\n",
    "            my_agent_goals, ai_agent_goals = play_match_with_dqn_agent(test_env)\n",
    "            total_goals_scored += my_agent_goals\n",
    "            total_goal_conceded += ai_agent_goals\n",
    "            if my_agent_goals > ai_agent_goals:\n",
    "                match_result = \"WIN!\"\n",
    "                win_count += 1\n",
    "            elif my_agent_goals < ai_agent_goals:\n",
    "                match_result = \"DEFEAT!\"\n",
    "            else:\n",
    "                match_result = \"DRAW\"\n",
    "                draw_count += 1\n",
    "            print(f\"Match {i+1}: {match_result.ljust(10, ' ')} | MY AGENT {my_agent_goals} - {ai_agent_goals} AI\")\n",
    "        print(f\"Results of {total_matches} Matches in {scenario_name} scenario:\")\n",
    "        print(f\"WON: {win_count}, LOST: {total_matches-win_count-draw_count}, DREW: {draw_count} | total goals scored: {total_goals_scored}, total goals conceded: {total_goal_conceded}\\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}