mo_gymnasium_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ed73b6a4",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/Farama-Foundation/MO-Gymnasium/blob/main/mo_gymnasium_demo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab7f2ffc",
   "metadata": {
    "id": "ab7f2ffc"
   },
   "source": [
    "## Step 1: install and import the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "o8lMdzzrRf0O",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "o8lMdzzrRf0O",
    "outputId": "b8bb653e-a81f-42b7-e251-40179a2d0d76"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'MO-Gymnasium'...\n",
      "remote: Enumerating objects: 1343, done.\u001b[K\n",
      "remote: Counting objects: 100% (645/645), done.\u001b[K\n",
      "remote: Compressing objects: 100% (395/395), done.\u001b[K\n",
      "remote: Total 1343 (delta 321), reused 491 (delta 211), pack-reused 698\u001b[K\n",
      "Receiving objects: 100% (1343/1343), 2.34 MiB | 1.79 MiB/s, done.\n",
      "Resolving deltas: 100% (663/663), done.\n",
      "Obtaining file:///Users/florian.felten/Documents/MO-Gymnasium/mo-gymnasium\n",
      "  Installing build dependencies ... \u001b[?25ldone\n",
      "\u001b[?25h  Checking if build backend supports build_editable ... \u001b[?25ldone\n",
      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
      "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: pygame>=2.1.0 in /usr/local/lib/python3.10/site-packages (from mo-gym==0.3.0) (2.1.0)\n",
      "Requirement already satisfied: scipy>=1.7.3 in /usr/local/lib/python3.10/site-packages (from mo-gym==0.3.0) (1.9.1)\n",
      "Requirement already satisfied: pymoo>=0.6.0 in /usr/local/lib/python3.10/site-packages (from mo-gym==0.3.0) (0.6.0)\n",
      "Requirement already satisfied: numpy>=1.21.0 in /usr/local/lib/python3.10/site-packages (from mo-gym==0.3.0) (1.23.3)\n",
      "Requirement already satisfied: gym>=0.26 in /usr/local/lib/python3.10/site-packages (from mo-gym==0.3.0) (0.26.2)\n",
      "Requirement already satisfied: gym-notices>=0.0.4 in /usr/local/lib/python3.10/site-packages (from gym>=0.26->mo-gym==0.3.0) (0.0.8)\n",
      "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.10/site-packages (from gym>=0.26->mo-gym==0.3.0) (2.2.0)\n",
      "Requirement already satisfied: autograd>=1.4 in /usr/local/lib/python3.10/site-packages (from pymoo>=0.6.0->mo-gym==0.3.0) (1.5)\n",
      "Requirement already satisfied: alive-progress in /usr/local/lib/python3.10/site-packages (from pymoo>=0.6.0->mo-gym==0.3.0) (2.4.1)\n",
      "Requirement already satisfied: matplotlib>=3 in /usr/local/lib/python3.10/site-packages (from pymoo>=0.6.0->mo-gym==0.3.0) (3.6.0)\n",
      "Requirement already satisfied: Deprecated in /usr/local/lib/python3.10/site-packages (from pymoo>=0.6.0->mo-gym==0.3.0) (1.2.13)\n",
      "Requirement already satisfied: cma==3.2.2 in /usr/local/lib/python3.10/site-packages (from pymoo>=0.6.0->mo-gym==0.3.0) (3.2.2)\n",
      "Requirement already satisfied: dill in /usr/local/lib/python3.10/site-packages (from pymoo>=0.6.0->mo-gym==0.3.0) (0.3.5.1)\n",
      "Requirement already satisfied: future>=0.15.2 in /usr/local/lib/python3.10/site-packages (from autograd>=1.4->pymoo>=0.6.0->mo-gym==0.3.0) (0.18.2)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/site-packages (from matplotlib>=3->pymoo>=0.6.0->mo-gym==0.3.0) (1.0.5)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/site-packages (from matplotlib>=3->pymoo>=0.6.0->mo-gym==0.3.0) (0.11.0)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/site-packages (from matplotlib>=3->pymoo>=0.6.0->mo-gym==0.3.0) (9.2.0)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /usr/local/lib/python3.10/site-packages (from matplotlib>=3->pymoo>=0.6.0->mo-gym==0.3.0) (3.0.9)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/site-packages (from matplotlib>=3->pymoo>=0.6.0->mo-gym==0.3.0) (2.8.2)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/site-packages (from matplotlib>=3->pymoo>=0.6.0->mo-gym==0.3.0) (1.4.4)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/site-packages (from matplotlib>=3->pymoo>=0.6.0->mo-gym==0.3.0) (21.3)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/site-packages (from matplotlib>=3->pymoo>=0.6.0->mo-gym==0.3.0) (4.37.3)\n",
      "Requirement already satisfied: grapheme==0.6.0 in /usr/local/lib/python3.10/site-packages (from alive-progress->pymoo>=0.6.0->mo-gym==0.3.0) (0.6.0)\n",
      "Requirement already satisfied: about-time==3.1.1 in /usr/local/lib/python3.10/site-packages (from alive-progress->pymoo>=0.6.0->mo-gym==0.3.0) (3.1.1)\n",
      "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.10/site-packages (from Deprecated->pymoo>=0.6.0->mo-gym==0.3.0) (1.14.1)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib>=3->pymoo>=0.6.0->mo-gym==0.3.0) (1.16.0)\n",
      "Installing collected packages: mo-gym\n",
      "  Attempting uninstall: mo-gym\n",
      "    Found existing installation: mo-gym 0.3.0\n",
      "    Uninstalling mo-gym-0.3.0:\n",
      "      Successfully uninstalled mo-gym-0.3.0\n",
      "  Running setup.py develop for mo-gym\n",
      "Successfully installed mo-gym-0.3.0\n"
     ]
    }
   ],
   "source": [
    "!pip install mo-gymnasium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "43bddbf7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting moviepy\n",
      "  Downloading moviepy-1.0.3.tar.gz (388 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m388.3/388.3 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting decorator<5.0,>=4.0.2 (from moviepy)\n",
      "  Using cached decorator-4.4.2-py2.py3-none-any.whl (9.2 kB)\n",
      "Requirement already satisfied: tqdm<5.0,>=4.11.2 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from moviepy) (4.62.3)\n",
      "Requirement already satisfied: requests<3.0,>=2.8.1 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from moviepy) (2.26.0)\n",
      "Collecting proglog<=1.0.0 (from moviepy)\n",
      "  Downloading proglog-0.1.10-py3-none-any.whl (6.1 kB)\n",
      "Requirement already satisfied: numpy>=1.17.3 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from moviepy) (1.25.1)\n",
      "Requirement already satisfied: imageio<3.0,>=2.5 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from moviepy) (2.19.2)\n",
      "Collecting imageio_ffmpeg>=0.2.0 (from moviepy)\n",
      "  Downloading imageio_ffmpeg-0.4.8-py3-none-manylinux2010_x86_64.whl (26.9 MB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m26.9/26.9 MB\u001b[0m \u001b[31m16.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m0:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pillow>=8.3.2 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from imageio<3.0,>=2.5->moviepy) (8.4.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from requests<3.0,>=2.8.1->moviepy) (1.26.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from requests<3.0,>=2.8.1->moviepy) (2021.10.8)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from requests<3.0,>=2.8.1->moviepy) (2.0.7)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from requests<3.0,>=2.8.1->moviepy) (3.3)\n",
      "Building wheels for collected packages: moviepy\n",
      "  Building wheel for moviepy (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for moviepy: filename=moviepy-1.0.3-py3-none-any.whl size=110744 sha256=ebfb73108e06c510590c09d7342c453afbe3c16206070110f5707d8707ac295e\n",
      "  Stored in directory: /home/lucas/.cache/pip/wheels/29/15/e4/4f790bec6acd51a00b67e8ee1394f0bc6e0135c315f8ff399a\n",
      "Successfully built moviepy\n",
      "Installing collected packages: proglog, imageio_ffmpeg, decorator, moviepy\n",
      "  Attempting uninstall: decorator\n",
      "    Found existing installation: decorator 5.1.0\n",
      "    Uninstalling decorator-5.1.0:\n",
      "      Successfully uninstalled decorator-5.1.0\n",
      "Successfully installed decorator-4.4.2 imageio_ffmpeg-0.4.8 moviepy-1.0.3 proglog-0.1.10\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "!{sys.executable} -m pip install moviepy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "846b9364",
   "metadata": {
    "id": "846b9364"
   },
   "outputs": [],
   "source": [
    "import gymnasium as gym\n",
    "import mo_gymnasium as mo_gym"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4fef046",
   "metadata": {
    "id": "f4fef046"
   },
   "source": [
    "## Step 2: create an environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d5415397",
   "metadata": {
    "id": "d5415397"
   },
   "outputs": [],
   "source": [
    "env = mo_gym.make(\"mo-mountaincar-v0\", render_mode=\"rgb_array\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "870ae1a4",
   "metadata": {
    "id": "870ae1a4"
   },
   "source": [
    "## Step 3: extract environment information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8db784fa",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8db784fa",
    "outputId": "dd097381-1f1b-48c3-8be2-f284aa9f493e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Box([-1.2  -0.07], [0.6  0.07], (2,), float32)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.observation_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8e8d91c7",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8e8d91c7",
    "outputId": "3c51452a-8713-4599-9064-72fff4faca85"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Discrete(3)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.action_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5bff02b6",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5bff02b6",
    "outputId": "fe8ee38e-ff94-4ad2-9d9c-e2cc58d89159"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages/gymnasium/core.py:297: UserWarning: \u001b[33mWARN: env.reward_space to get variables from other wrappers is deprecated and will be removed in v1.0, to get this variable you can do `env.unwrapped.reward_space` for environment variables or `env.get_attr('reward_space')` that will search the reminding wrappers.\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Box(-1.0, [-1.  0.  0.], (3,), float32)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.reward_space"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "f01faaa2",
   "metadata": {
    "id": "f01faaa2"
   },
   "source": [
    "## Step 4: use Gymnasium features in MO-Gymnasium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c64e5417",
   "metadata": {
    "id": "c64e5417"
   },
   "outputs": [],
   "source": [
    "from gymnasium.wrappers.record_video import RecordVideo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a4190496",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "a4190496",
    "outputId": "7739e628-5ef1-4086-98df-33ff30af0a29"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages/gymnasium/wrappers/record_video.py:94: UserWarning: \u001b[33mWARN: Overwriting existing videos at /home/lucas/Desktop/MO-Gymnasium/videos/demo folder (try specifying a different `video_folder` for the `RecordVideo` wrapper if this is not desired)\u001b[0m\n",
      "  logger.warn(\n",
      "/home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages/gymnasium/core.py:297: UserWarning: \u001b[33mWARN: env.is_vector_env to get variables from other wrappers is deprecated and will be removed in v1.0, to get this variable you can do `env.unwrapped.is_vector_env` for environment variables or `env.get_attr('is_vector_env')` that will search the reminding wrappers.\u001b[0m\n",
      "  logger.warn(\n"
     ]
    }
   ],
   "source": [
    "env = RecordVideo(env, \"videos/demo\", episode_trigger=lambda e: True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbde25c4",
   "metadata": {
    "id": "dbde25c4"
   },
   "source": [
    "## Step 5: go through the environment with a random agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "066db701",
   "metadata": {
    "id": "066db701"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Moviepy - Building video /home/lucas/Desktop/MO-Gymnasium/videos/demo/rl-video-episode-0.mp4.\n",
      "Moviepy - Writing video /home/lucas/Desktop/MO-Gymnasium/videos/demo/rl-video-episode-0.mp4\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                               "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Moviepy - Done !\n",
      "Moviepy - video ready /home/lucas/Desktop/MO-Gymnasium/videos/demo/rl-video-episode-0.mp4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "env.reset()\n",
    "done = False\n",
    "\n",
    "while not done:\n",
    "    obs, vec_reward, terminated, truncated, info = env.step(env.action_space.sample())\n",
    "    done = terminated or truncated"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "IG7S7nx3cces",
   "metadata": {
    "id": "IG7S7nx3cces"
   },
   "source": [
    "## Step 6: scalarize env and run it with stable-baselines3\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "DHkO3R7uccOy",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DHkO3R7uccOy",
    "outputId": "7dcaa4a2-f566-40e8-f62a-12d4320c3df4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: stable-baselines3 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (1.5.0)\n",
      "Collecting stable-baselines3\n",
      "  Obtaining dependency information for stable-baselines3 from https://files.pythonhosted.org/packages/d9/57/13d4e4b7bbbc940815964ac31e205263b8133f1f2a0147bd4ca884a6e174/stable_baselines3-2.0.0-py3-none-any.whl.metadata\n",
      "  Using cached stable_baselines3-2.0.0-py3-none-any.whl.metadata (5.4 kB)\n",
      "Requirement already satisfied: gymnasium==0.28.1 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from stable-baselines3) (0.28.1)\n",
      "Requirement already satisfied: numpy>=1.20 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from stable-baselines3) (1.25.1)\n",
      "Collecting torch>=1.11 (from stable-baselines3)\n",
      "  Using cached torch-2.0.1-cp39-cp39-manylinux1_x86_64.whl (619.9 MB)\n",
      "Requirement already satisfied: cloudpickle in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from stable-baselines3) (2.0.0)\n",
      "Requirement already satisfied: pandas in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from stable-baselines3) (1.3.4)\n",
      "Requirement already satisfied: matplotlib in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from stable-baselines3) (3.5.1)\n",
      "Requirement already satisfied: jax-jumpy>=1.0.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from gymnasium==0.28.1->stable-baselines3) (1.0.0)\n",
      "Requirement already satisfied: typing-extensions>=4.3.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from gymnasium==0.28.1->stable-baselines3) (4.7.1)\n",
      "Requirement already satisfied: farama-notifications>=0.0.1 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from gymnasium==0.28.1->stable-baselines3) (0.0.4)\n",
      "Requirement already satisfied: importlib-metadata>=4.8.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from gymnasium==0.28.1->stable-baselines3) (4.11.2)\n",
      "Requirement already satisfied: filelock in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (3.4.0)\n",
      "Requirement already satisfied: sympy in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (1.12)\n",
      "Requirement already satisfied: networkx in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (2.6.3)\n",
      "Requirement already satisfied: jinja2 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (3.0.3)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (11.7.99)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (11.7.99)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (11.7.101)\n",
      "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (8.5.0.96)\n",
      "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (11.10.3.66)\n",
      "Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (10.9.0.58)\n",
      "Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (10.2.10.91)\n",
      "Requirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (11.4.0.1)\n",
      "Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (11.7.4.91)\n",
      "Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (2.14.3)\n",
      "Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (11.7.91)\n",
      "Requirement already satisfied: triton==2.0.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (2.0.0)\n",
      "Requirement already satisfied: setuptools in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.11->stable-baselines3) (58.0.4)\n",
      "Requirement already satisfied: wheel in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.11->stable-baselines3) (0.37.0)\n",
      "Requirement already satisfied: cmake in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.11->stable-baselines3) (3.27.0)\n",
      "Requirement already satisfied: lit in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.11->stable-baselines3) (16.0.6)\n",
      "Requirement already satisfied: cycler>=0.10 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from matplotlib->stable-baselines3) (0.10.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from matplotlib->stable-baselines3) (4.28.5)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from matplotlib->stable-baselines3) (1.3.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from matplotlib->stable-baselines3) (21.3)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from matplotlib->stable-baselines3) (8.4.0)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from matplotlib->stable-baselines3) (2.4.7)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from matplotlib->stable-baselines3) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from pandas->stable-baselines3) (2021.3)\n",
      "Requirement already satisfied: six in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from cycler>=0.10->matplotlib->stable-baselines3) (1.16.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from importlib-metadata>=4.8.0->gymnasium==0.28.1->stable-baselines3) (3.7.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from jinja2->torch>=1.11->stable-baselines3) (2.1.0)\n",
      "Requirement already satisfied: mpmath>=0.19 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from sympy->torch>=1.11->stable-baselines3) (1.3.0)\n",
      "Using cached stable_baselines3-2.0.0-py3-none-any.whl (178 kB)\n",
      "Installing collected packages: torch, stable-baselines3\n",
      "  Attempting uninstall: stable-baselines3\n",
      "    Found existing installation: stable-baselines3 1.5.0\n",
      "    Uninstalling stable-baselines3-1.5.0:\n",
      "      Successfully uninstalled stable-baselines3-1.5.0\n",
      "Successfully installed stable-baselines3-2.0.0 torch-2.0.1\n"
     ]
    }
   ],
   "source": [
    "!pip install stable-baselines3 --upgrade"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "lgPC1qU8c-0A",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "lgPC1qU8c-0A",
    "outputId": "f3b8d2a4-0a91-4005-c0e9-64d9f34f19d3"
   },
   "outputs": [],
   "source": [
    "import stable_baselines3 as sb3\n",
    "import numpy as np\n",
    "import mo_gymnasium as mo_gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "mn7dMweScz1U",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "mn7dMweScz1U",
    "outputId": "fe80d623-a46d-4d9e-bd27-75188c12aa19"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages/gymnasium/core.py:297: UserWarning: \u001b[33mWARN: env.reward_space to get variables from other wrappers is deprecated and will be removed in v1.0, to get this variable you can do `env.unwrapped.reward_space` for environment variables or `env.get_attr('reward_space')` that will search the reminding wrappers.\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.dqn.dqn.DQN at 0x7f9b66f70ee0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Linear scalarizes the environment\n",
    "env = mo_gym.LinearReward(mo_gym.make(\"mo-mountaincar-v0\"), weight=np.array([0.9, 0.1, 0.0]))\n",
    "\n",
    "# Run DQN agent!\n",
    "agent = sb3.DQN(\"MlpPolicy\", env)\n",
    "agent.learn(10000)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "include_colab_link": true,
   "name": "mo-gym-demo.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.14"
  },
  "vscode": {
   "interpreter": {
    "hash": "ac2eaa0ea0ebeafcc7822e65e46aa9d4f966f30b695406963e145ea4a91cd4fc"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}