Q - Learning Frozen Lake.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing libraries\n",
    "import numpy as np\n",
    "import gym\n",
    "import random\n",
    "import time\n",
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#envt set up\n",
    "env = gym.make(\"FrozenLake-v0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "action_size = env.action_space.n\n",
    "state_size = env.observation_space.n\n",
    "\n",
    "q_table = np.zeros((state_size, action_size))\n",
    "print(q_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_episodes = 10000\n",
    "max_steps_per_episode = 100\n",
    "\n",
    "learning_rate = 0.1\n",
    "discount_rate = 0.99\n",
    "\n",
    "exploration_rate = 1\n",
    "max_exploration_rate = 1\n",
    "min_exploration_rate = 0.01\n",
    "exploration_decay_rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average reward per thousand episodes\n",
      "\n",
      "1000 :  0.04000000000000003\n",
      "2000 :  0.19100000000000014\n",
      "3000 :  0.4120000000000003\n",
      "4000 :  0.5710000000000004\n",
      "5000 :  0.6450000000000005\n",
      "6000 :  0.6710000000000005\n",
      "7000 :  0.6870000000000005\n",
      "8000 :  0.6630000000000005\n",
      "9000 :  0.6840000000000005\n",
      "10000 :  0.6730000000000005\n"
     ]
    }
   ],
   "source": [
    "rewards_all_episodes = []  #hold all the rewards from each episode\n",
    "#q learning algo\n",
    "for episode in range(num_episodes): #every thing that happens in a episode\n",
    "    state = env.reset() #reset the state of envt to start\n",
    "    done = False #track if the episode is finished\n",
    "    rewards_current_episode = 0 \n",
    "    \n",
    "    for step in range(max_steps_per_episode): #every thing that happens in a single time step in a episode\n",
    "\n",
    "    # Exploration-exploitation trade-off\n",
    "        exploration_rate_threshold = random.uniform(0, 1) #whether the agent eill explore \n",
    "        if exploration_rate_threshold > exploration_rate:\n",
    "            action = np.argmax(q_table[state,:]) # exploit envt\n",
    "        else:\n",
    "            action = env.action_space.sample() #explore\n",
    "        \n",
    "        # action is choosen\n",
    "        new_state, reward, done, info = env.step(action)#execute the action\n",
    "        \n",
    "        #update q table for Q(s,a)\n",
    "        q_table[state, action] = q_table[state, action] * (1 - learning_rate) + \\\n",
    "            learning_rate * (reward + discount_rate * np.max(q_table[new_state, :]))\n",
    "        state = new_state #set current state to new\n",
    "        rewards_current_episode += reward #update the reward\n",
    "        if done == True: #check if last action ended the episode\n",
    "            break\n",
    "            \n",
    "    exploration_rate = min_exploration_rate + \\\n",
    "    (max_exploration_rate - min_exploration_rate) * np.exp(-exploration_decay_rate*episode) #decay exploration rate\n",
    "    rewards_all_episodes.append(rewards_current_episode) #append the episode reward\n",
    "    \n",
    "# Calculate and print the average reward per thousand episodes\n",
    "rewards_per_thosand_episodes = np.split(np.array(rewards_all_episodes),num_episodes/1000)\n",
    "count = 1000\n",
    "\n",
    "print(\"Average reward per thousand episodes\\n\")\n",
    "for r in rewards_per_thosand_episodes:\n",
    "    print(count, \": \", str(sum(r/1000)))\n",
    "    count += 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (Down)\n",
      "SFFF\n",
      "FHFH\n",
      "FFFH\n",
      "HFF\u001b[41mG\u001b[0m\n",
      "You reached the goal\n"
     ]
    }
   ],
   "source": [
    "#we will play 3 episodes of the game\n",
    "for episode in range(3):\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    print(\"EPISODE \", episode+1, \"\\n\\n\\n\\n\")\n",
    "    time.sleep(1)\n",
    "\n",
    "    for step in range(max_steps_per_episode):\n",
    "        clear_output(wait=True)\n",
    "        env.render()\n",
    "        time.sleep(0.3)\n",
    "        action = np.argmax(q_table[state,:])        \n",
    "        new_state, reward, done, info = env.step(action)  \n",
    "        if done:\n",
    "            clear_output(wait=True)\n",
    "            env.render()\n",
    "            if reward == 1:\n",
    "                print(\"You reached the goal\")\n",
    "                time.sleep(3)\n",
    "            else:\n",
    "                print(\"You fell through a hole\")\n",
    "                time.sleep(3)\n",
    "                clear_output(wait=True)\n",
    "            break\n",
    "        state = new_state            \n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}