openai-gym-env-taxi-v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# OpenAI `gym` env `Taxi-v2`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-10-18 15:15:59,660] Making new env: Taxi-v2\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('Taxi-v2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current state: 293\n"
     ]
    }
   ],
   "source": [
    "print('Current state: {:,}'.format(env.reset()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Possible states: 500\n",
      "Possible actions: 6\n"
     ]
    }
   ],
   "source": [
    "print('Possible states: {:,}'.format(env.observation_space.n))\n",
    "print('Possible actions: {:,}'.format(env.action_space.n))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+\n",
      "|R: | : :\u001b[35mG\u001b[0m|\n",
      "| : : : : |\n",
      "| : : : :\u001b[43m \u001b[0m|\n",
      "| | : | : |\n",
      "|Y| : |\u001b[34;1mB\u001b[0m: |\n",
      "+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "env.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# setting the current state to 114\n",
    "env.env.s = 114"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+\n",
      "|R: | : :G|\n",
      "|\u001b[43m \u001b[0m: : : : |\n",
      "| : : : : |\n",
      "| | : | : |\n",
      "|\u001b[35mY\u001b[0m| : |\u001b[34;1mB\u001b[0m: |\n",
      "+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "env.render()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Possible actions\n",
    "\n",
    "* 0 - down (south)\n",
    "* 1 - up (north)\n",
    "* 2 - right (east)\n",
    "* 3 - left (west) \n",
    "* 4 - pickup\n",
    "* 5 - dropoff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14, -1, False, {'prob': 1.0})"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.step(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+\n",
      "|\u001b[43mR\u001b[0m: | : :G|\n",
      "| : : : : |\n",
      "| : : : : |\n",
      "| | : | : |\n",
      "|\u001b[35mY\u001b[0m| : |\u001b[34;1mB\u001b[0m: |\n",
      "+---------+\n",
      "  (North)\n"
     ]
    }
   ],
   "source": [
    "env.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(114, -10, False, {'prob': 1.0})"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.step(0)  # move down\n",
    "env.step(4)  # pick up passanger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+\n",
      "|R: | : :G|\n",
      "|\u001b[43m \u001b[0m: : : : |\n",
      "| : : : : |\n",
      "| | : | : |\n",
      "|\u001b[35mY\u001b[0m| : |\u001b[34;1mB\u001b[0m: |\n",
      "+---------+\n",
      "  (Pickup)\n"
     ]
    }
   ],
   "source": [
    "env.render()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Brute force solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+\n",
      "|R: | : :\u001b[35m\u001b[42mG\u001b[0m\u001b[0m|\n",
      "| : : : : |\n",
      "| : : : : |\n",
      "| | : | : |\n",
      "|Y| : |B: |\n",
      "+---------+\n",
      "  (Dropoff)\n"
     ]
    }
   ],
   "source": [
    "state = env.reset()\n",
    "total_rewards, reward = 0, None\n",
    "counter = 0\n",
    "\n",
    "while reward != 20:\n",
    "    state, reward, done, info = env.step(env.action_space.sample())\n",
    "    counter += 1  # increment the counter\n",
    "    total_rewards += reward  # accumulated reward\n",
    "    if counter >= 5000:   # terminate after 5000 episodes\n",
    "        print('Giving up after {:,} episodes.'.format(counter))\n",
    "        break\n",
    "\n",
    "env.render()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Q Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(500, 6)\n"
     ]
    }
   ],
   "source": [
    "Q = np.zeros(shape=[env.observation_space.n, env.action_space.n])\n",
    "print(Q.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "G = 0  # total accumulated reward\n",
    "alpha = 0.618  # learning rate (Golden ratio)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Basic Q Learning Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 50\tAccumulated reward = 2\n",
      "Episode 100\tAccumulated reward = -94\n",
      "Episode 150\tAccumulated reward = -30\n",
      "Episode 200\tAccumulated reward = 9\n",
      "Episode 250\tAccumulated reward = 8\n",
      "Episode 300\tAccumulated reward = -6\n",
      "Episode 350\tAccumulated reward = 8\n",
      "Episode 400\tAccumulated reward = 1\n",
      "Episode 450\tAccumulated reward = 14\n",
      "Episode 500\tAccumulated reward = -2\n",
      "Episode 550\tAccumulated reward = 6\n",
      "Episode 600\tAccumulated reward = 5\n",
      "Episode 650\tAccumulated reward = 4\n",
      "Episode 700\tAccumulated reward = 8\n",
      "Episode 750\tAccumulated reward = 6\n",
      "Episode 800\tAccumulated reward = 11\n",
      "Episode 850\tAccumulated reward = 8\n",
      "Episode 900\tAccumulated reward = 5\n",
      "Episode 950\tAccumulated reward = 6\n",
      "+---------+\n",
      "|R: | : :G|\n",
      "| : : : : |\n",
      "| : : : : |\n",
      "| | : | : |\n",
      "|\u001b[35m\u001b[42mY\u001b[0m\u001b[0m| : |B: |\n",
      "+---------+\n",
      "  (Dropoff)\n"
     ]
    }
   ],
   "source": [
    "episodes = 1000\n",
    "for episode in range(1, episodes):\n",
    "    done = None\n",
    "    G, reward = 0, None\n",
    "    state = env.reset()\n",
    "    while done != True:  # as long as we ain't done\n",
    "        action = np.argmax(Q[state])   # select action from Q\n",
    "        state2, reward, done, info = env.step(action)  # take that action\n",
    "        Q[state, action] += alpha * (reward + np.max(Q[state2]) - Q[state, action])\n",
    "        G += reward\n",
    "        state = state2\n",
    "    if episode % 50 == 0:\n",
    "        print('Episode {:,}\\tAccumulated reward = {:,}'.format(episode, G))\n",
    "env.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}