Project_4_Blackjack.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import gymnasium as gym\n",
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.vec_env import DummyVecEnv, VecTransposeImage, VecFrameStack\n",
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "from stable_baselines3.common.env_util import make_vec_env"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test enviroment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "enviroment_name = 'Blackjack-v1'\n",
    "env = gym.make(enviroment_name, render_mode='human')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((13, 3, 1), {})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Discrete(2)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 0: stick, 1: hit\n",
    "env.action_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Tuple(Discrete(32), Discrete(11), Discrete(2))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The player’s current sum, the value of the dealer’s one showing card (1-10 where 1 is ace), and whether the player holds a usable ace (0 or 1)\n",
    "env.observation_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17, 5, 1)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.observation_space.sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 1 Score -1.0\n",
      "Episode: 2 Score -1.0\n",
      "Episode: 3 Score 1.0\n",
      "Episode: 4 Score -1.0\n",
      "Episode: 5 Score -1.0\n"
     ]
    }
   ],
   "source": [
    "episodes = 5\n",
    "for episode in range(1, episodes+1):\n",
    "  obs = env.reset()\n",
    "  done = False\n",
    "  score = 0\n",
    "\n",
    "  while not done:\n",
    "    env.render()\n",
    "    action = env.action_space.sample()\n",
    "    obs, reward, done, truncated, info = env.step(action)\n",
    "    score += reward\n",
    "\n",
    "  print('Episode: {} Score {}'.format(episode, score))\n",
    "#env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make(enviroment_name, render_mode='human')\n",
    "env = gym.wrappers.FlattenObservation(env)\n",
    "env = DummyVecEnv([lambda: env])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cpu device\n"
     ]
    }
   ],
   "source": [
    "log_path = os.path.join('Training', 'Logs')\n",
    "model = PPO('MlpPolicy', env, verbose=1, tensorboard_log=log_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logging to Training\\Logs\\PPO_7\n",
      "-----------------------------\n",
      "| time/              |      |\n",
      "|    fps             | 2    |\n",
      "|    iterations      | 1    |\n",
      "|    time_elapsed    | 888  |\n",
      "|    total_timesteps | 2048 |\n",
      "-----------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 2           |\n",
      "|    iterations           | 2           |\n",
      "|    time_elapsed         | 1800        |\n",
      "|    total_timesteps      | 4096        |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.015928742 |\n",
      "|    clip_fraction        | 0.289       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.677      |\n",
      "|    explained_variance   | -0.169      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.319       |\n",
      "|    n_updates            | 10          |\n",
      "|    policy_gradient_loss | -0.0456     |\n",
      "|    value_loss           | 0.747       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 2           |\n",
      "|    iterations           | 3           |\n",
      "|    time_elapsed         | 2712        |\n",
      "|    total_timesteps      | 6144        |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020384211 |\n",
      "|    clip_fraction        | 0.307       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.627      |\n",
      "|    explained_variance   | 0.0812      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.297       |\n",
      "|    n_updates            | 20          |\n",
      "|    policy_gradient_loss | -0.0508     |\n",
      "|    value_loss           | 0.744       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 2           |\n",
      "|    iterations           | 4           |\n",
      "|    time_elapsed         | 3637        |\n",
      "|    total_timesteps      | 8192        |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.032364704 |\n",
      "|    clip_fraction        | 0.212       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.555      |\n",
      "|    explained_variance   | 0.121       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.276       |\n",
      "|    n_updates            | 30          |\n",
      "|    policy_gradient_loss | -0.0359     |\n",
      "|    value_loss           | 0.742       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 2           |\n",
      "|    iterations           | 5           |\n",
      "|    time_elapsed         | 4555        |\n",
      "|    total_timesteps      | 10240       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016928785 |\n",
      "|    clip_fraction        | 0.117       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.505      |\n",
      "|    explained_variance   | 0.15        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.323       |\n",
      "|    n_updates            | 40          |\n",
      "|    policy_gradient_loss | -0.0194     |\n",
      "|    value_loss           | 0.714       |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x215f9cdc490>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.learn(total_timesteps=10000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "Lunar_path = os.path.join('Training', 'Saved models', 'PPO_Blackjack_Model')\n",
    "model.save(Lunar_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "del model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = PPO.load(Lunar_path, env)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\william.mendes\\Anaconda3\\lib\\site-packages\\stable_baselines3\\common\\evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean_reward:0.00 +/- 1.00\n"
     ]
    }
   ],
   "source": [
    "mean_reward, std_reward = evaluate_policy(model, env, n_eval_episodes=10, render=True)\n",
    "print(f\"mean_reward:{mean_reward:.2f} +/- {std_reward:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "episodes = 5\n",
    "for episode in range(1, episodes+1):\n",
    "  obs = env.reset()[0]\n",
    "  done = False\n",
    "  score = 0\n",
    "\n",
    "  while not done:\n",
    "    env.render()\n",
    "    action = model.predict(obs)[0]\n",
    "    obs, reward, done, truncated, info = env.step(action)\n",
    "    score += reward\n",
    "\n",
    "  print('Episode: {} Score {}'.format(episode, score))\n",
    "#env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}