HandsOn-WindyGridWorld environment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6xF7RMruQeyv"
   },
   "source": [
    "### gridworld 4*4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DObA9QJUQey3"
   },
   "source": [
    "\n",
    "    This is a WindyGrid World environment.\n",
    "    You are an agent on an MxN grid and your goal is to reach the terminal\n",
    "    state at the top left or the bottom right corner.\n",
    "    \n",
    "\n",
    "    For example, a 4x4 grid looks as follows:\n",
    "\n",
    "    T  o  o  o\n",
    "    o  x  o  o\n",
    "    o  o  o  o\n",
    "    o  o  o  T\n",
    "\n",
    "    x is your position and T are the two terminal states.\n",
    "\n",
    "    You can take actions in each direction (UP=0, RIGHT=1, DOWN=2, LEFT=3).\n",
    "    Actions going off the edge leave you in your current state.\n",
    "    You receive a reward of -1 at each step until you reach a terminal state.\n",
    "    Notice : the wind is blowing , so you won't always move in the direction you intend.\n",
    "    \"\"\"\"\"\"You dont need to use the AMAlearn package on this hands on session\"\"\"\"\"\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-0-0tQGeQey6"
   },
   "source": [
    "### Getting Started"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qGhP1kPQQey7"
   },
   "source": [
    "Run the following code and observe the result."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "9Gr3FsSDQey9"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "Aw98C-chRQiC"
   },
   "outputs": [],
   "source": [
    "import io\n",
    "import numpy as np\n",
    "import sys\n",
    "from gym.envs.toy_text import discrete\n",
    "\n",
    "UP = 0\n",
    "RIGHT = 1\n",
    "DOWN = 2\n",
    "LEFT = 3\n",
    "\n",
    "class WindyGridworldEnv(discrete.DiscreteEnv):\n",
    "    \"\"\"\n",
    "    This is Grid World environment. \n",
    "    You are an agent on an MxN grid and your goal is to reach the terminal\n",
    "    state at the top left or the bottom right corner.\n",
    "\n",
    "    For example, a 4x4 grid looks as follows:\n",
    "\n",
    "    T  o  o  o\n",
    "    o  x  o  o\n",
    "    o  o  o  o\n",
    "    o  o  o  T\n",
    "\n",
    "    x is your position and T are the two terminal states.\n",
    "\n",
    "    You can take actions in each direction (UP=0, RIGHT=1, DOWN=2, LEFT=3).\n",
    "    Actions going off the edge leave you in your current state.\n",
    "    You receive a reward of -1 at each step until you reach a terminal state.\n",
    "    Notice : the wind is blowing , so you won't always move in the direction you intend.\n",
    "    \"\"\"\n",
    "\n",
    "    metadata = {'render.modes': ['human', 'ansi']}\n",
    "\n",
    "    def __init__(self, shape=[4,4]):\n",
    "        if not isinstance(shape, (list, tuple)) or not len(shape) == 2:\n",
    "            raise ValueError('shape argument must be a list/tuple of length 2')\n",
    "\n",
    "        self.shape = shape\n",
    "\n",
    "        nS = np.prod(shape)\n",
    "        nA = 4\n",
    "\n",
    "        MAX_Y = shape[0]\n",
    "        MAX_X = shape[1]\n",
    "\n",
    "        P = {s : {a : [] for a in range(nA)} for s in range(nS)}\n",
    "        grid = np.arange(nS).reshape(shape)\n",
    "        it = np.nditer(grid, flags=['multi_index'])\n",
    "\n",
    "        while not it.finished:\n",
    "            s = it.iterindex\n",
    "            y, x = it.multi_index\n",
    "\n",
    "            # P[s][a] = (prob, next_state, reward, is_done)\n",
    "            P[s] = {a : [] for a in range(nA)}\n",
    "\n",
    "            is_done = lambda s: s == 0 or s == (nS - 1)\n",
    "            reward = 0.0 if is_done(s) else -1.0\n",
    "\n",
    "            # We're stuck in a terminal state\n",
    "            if is_done(s):\n",
    "                P[s][UP] = [(1.0, s, reward, True)]\n",
    "                P[s][RIGHT] = [(1.0, s, reward, True)]\n",
    "                P[s][DOWN] = [(1.0, s, reward, True)]\n",
    "                P[s][LEFT] = [(1.0, s, reward, True)]\n",
    "            # Not a terminal state\n",
    "            else:\n",
    "                ns_up = s if y == 0 else s - MAX_X\n",
    "                ns_right = s if x == (MAX_X - 1) else s + 1\n",
    "                ns_down = s if y == (MAX_Y - 1) else s + MAX_X\n",
    "                ns_left = s if x == 0 else s - 1\n",
    "                rand = np.random.uniform()\n",
    "                P[s][UP] = [(0.8, ns_up, reward, is_done(ns_up)),(0.2, ns_right, reward, is_done(ns_up))] \n",
    "                P[s][RIGHT] = [(1.0, ns_right, reward, is_done(ns_right))]\n",
    "                P[s][DOWN] = [(0.8, ns_down, reward, is_done(ns_down)),(0.2, ns_right, reward, is_done(ns_down))] \n",
    "                P[s][LEFT] = [(0.8, ns_left, reward, is_done(ns_left)),(0.2, s, reward, is_done(ns_left))] \n",
    "\n",
    "            it.iternext()\n",
    "\n",
    "        # Initial state distribution is uniform\n",
    "        isd = np.ones(nS) / nS\n",
    "\n",
    "        # We expose the model of the environment for educational purposes\n",
    "        # This should not be used in any model-free learning algorithm\n",
    "        self.P = P\n",
    "\n",
    "        super(WindyGridworldEnv, self).__init__(nS, nA, P, isd)\n",
    "\n",
    "    def _render(self, mode='human', close=False):\n",
    "        \"\"\" Renders the current gridworld layout\n",
    "\n",
    "         For example, a 4x4 grid with the mode=\"human\" looks like:\n",
    "            T  o  o  o\n",
    "            o  x  o  o\n",
    "            o  o  o  o\n",
    "            o  o  o  T\n",
    "        where x is your position and T are the two terminal states.\n",
    "        \"\"\"\n",
    "        if close:\n",
    "            return\n",
    "\n",
    "        outfile = io.StringIO() if mode == 'ansi' else sys.stdout\n",
    "\n",
    "        grid = np.arange(self.nS).reshape(self.shape)\n",
    "        it = np.nditer(grid, flags=['multi_index'])\n",
    "        while not it.finished:\n",
    "            s = it.iterindex\n",
    "            y, x = it.multi_index\n",
    "\n",
    "            if self.s == s:\n",
    "                output = \" x \"\n",
    "            elif s == 0 or s == self.nS - 1:\n",
    "                output = \" T \"\n",
    "            else:\n",
    "                output = \" o \"\n",
    "\n",
    "            if x == 0:\n",
    "                output = output.lstrip()\n",
    "            if x == self.shape[1] - 1:\n",
    "                output = output.rstrip()\n",
    "\n",
    "            outfile.write(output)\n",
    "\n",
    "            if x == self.shape[1] - 1:\n",
    "                outfile.write(\"\\n\")\n",
    "\n",
    "            it.iternext()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "10H-4gOtQezD",
    "outputId": "c143c482-c00b-40a3-a5a9-c8f677725ff0",
    "scrolled": true
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "T  o  o  o\no  o  o  o\no  x  o  o\no  o  o  T\n{'prob': 0.8}\nT  o  o  o\no  o  o  o\no  o  o  o\no  x  o  T\n{'prob': 0.8}\nT  o  o  o\no  o  o  o\no  o  o  o\nx  o  o  T\n{'prob': 0.8}\nT  o  o  o\no  o  o  o\no  o  o  o\nx  o  o  T\n{'prob': 1.0}\nT  o  o  o\no  o  o  o\no  o  o  o\no  x  o  T\n{'prob': 1.0}\nT  o  o  o\no  o  o  o\no  o  o  o\no  o  x  T\n{'prob': 0.2}\nT  o  o  o\no  o  o  o\no  o  o  o\no  o  x  T\n{'prob': 0.8}\nT  o  o  o\no  o  o  o\no  o  x  o\no  o  o  T\n{'prob': 0.8}\nT  o  o  o\no  o  o  o\no  x  o  o\no  o  o  T\n{'prob': 0.8}\nT  o  o  o\no  o  o  o\nx  o  o  o\no  o  o  T\n{'prob': 0.8}\n"
     ]
    }
   ],
   "source": [
    "#run WindyGridworld.ipynb\n",
    "\n",
    "env =  WindyGridworldEnv()\n",
    "env.reset()\n",
    "\n",
    "for _ in range(10):\n",
    "    env._render()\n",
    "    state, reward, done, info = env.step(env.action_space.sample()) # Take a random action\n",
    "    print(info)\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kz61tW4RQezL"
   },
   "source": [
    "What are the action space and the state space?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "XISomCV3QezN",
    "outputId": "91725088-520b-4b0b-8957-5d480aa63553"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Discrete(4)\nDiscrete(16)\n"
     ]
    }
   ],
   "source": [
    "print(env.action_space)\n",
    "print(env.observation_space)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lBiIQyyg2ue3"
   },
   "source": [
    "\n",
    "\n",
    "---\n",
    "\n",
    "The action space is number of actions we can take in each state.\n",
    "\n",
    "The state space is number of states we are allowed to be in as our current state.\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ez-tL2EuQezR",
    "outputId": "02d12ab6-7697-4f08-c740-86d605e4174d"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "4\n16\n"
     ]
    }
   ],
   "source": [
    "print(env.nA)\n",
    "\n",
    "print(env.nS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KwN7CS47QezV"
   },
   "source": [
    "What does the following code indicate?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yhrNfQFKQezW",
    "outputId": "71ab6715-cfbe-4716-a9d2-4d47ce10792d"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[(0.8, 11, -1.0, False), (0.2, 7, -1.0, False)]\n"
     ]
    }
   ],
   "source": [
    "print(env.P[7][2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vMlh98d5n0OJ"
   },
   "source": [
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "env.P represents the transition probabilities of the environment. env.P[7] is the transition probabilities of 8th state of our enviroment. \n",
    "env.P[7][2] is when we choose to go down.\n",
    "\n",
    "It contains information about the probability, next state, reward and whether or not our action is done.\n",
    "\n",
    "\"reward\" = 0.0 if we are stuck in the terminal state, else we will get a -1.0 reward. also, \"is done\" turns true when we are on the last state, terminal state.\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eMWftlZO2RSS"
   },
   "source": [
    "NOTE: i used given functions as methods inside my classes :)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rnpRVmP2Qezh"
   },
   "source": [
    "Find the best policy using the Value Iteration algorithm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "CuWMqq8RrEpi"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class ValueIteration(object):\n",
    "    def __init__(self, env, discount_factor, theta):\n",
    "        self.env = env\n",
    "        self.discount_factor = discount_factor\n",
    "        self.theta = theta\n",
    "        self.states_value = [0] * self.env.nS\n",
    "        self.policy = np.zeros([self.env.nS, self.env.nA])\n",
    "        self.best_actions = []\n",
    "        self.end_of_ep = False\n",
    "\n",
    "    def find_action_value(self, curr_state):\n",
    "        states_value = np.zeros(self.env.nA)\n",
    "        for action in range(self.env.nA):\n",
    "            \n",
    "            for p_s_ns, next_state, state_reward, done in env.P[curr_state][action]:\n",
    "                states_value[action] += p_s_ns * (state_reward + discount_factor * self.states_value[next_state])\n",
    "        return states_value\n",
    "                \n",
    "    def find_best_value(self, curr_state):\n",
    "        states_value = self.find_action_value(curr_state)\n",
    "        return np.max(states_value)\n",
    "    \n",
    "    def find_best_action(self, curr_state):\n",
    "        states_value = self.find_action_value(curr_state)\n",
    "        best_action = np.argmax(states_value)\n",
    "        self.best_actions.append(best_action)\n",
    "        return best_action\n",
    "\n",
    "    def value_iteration(self):\n",
    "        while True:\n",
    "            delta = 0\n",
    "            for state in range(self.env.nS):\n",
    "                v_temp = self.states_value[state]\n",
    "                self.states_value[state] = self.find_best_value(state)\n",
    "                delta = max(delta, np.abs(v_temp - self.states_value[state]))\n",
    "            if delta < self.theta:\n",
    "                    break\n",
    "    \n",
    "    def find_policy(self):\n",
    "        for state in range(self.env.nS):\n",
    "            best_action = self.find_best_action(state)\n",
    "            self.policy[state, best_action] = 1.0\n",
    "            \n",
    "        print(self.policy)\n",
    "        \n",
    "        \n",
    "    def print_best_actions(self):\n",
    "        print(self.best_actions)\n",
    "\n",
    "    def print_states_value(self):\n",
    "        print(np.round((np.array(self.states_value)),2))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Gv7U6V0RtInh",
    "outputId": "bb22eeec-9dee-40a5-8732-a3ed2f921b9a"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[[1. 0. 0. 0.]\n [0. 0. 0. 1.]\n [0. 0. 0. 1.]\n [0. 0. 1. 0.]\n [1. 0. 0. 0.]\n [1. 0. 0. 0.]\n [0. 0. 1. 0.]\n [0. 0. 1. 0.]\n [1. 0. 0. 0.]\n [0. 0. 1. 0.]\n [0. 0. 1. 0.]\n [0. 0. 1. 0.]\n [0. 1. 0. 0.]\n [0. 1. 0. 0.]\n [0. 1. 0. 0.]\n [1. 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "discount_factor=1.0\n",
    "theta=0.0001\n",
    "my_value_iterator = ValueIteration(env, discount_factor, theta)\n",
    "my_value_iterator.value_iteration()\n",
    "my_value_iterator.find_policy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6e1N_ocVQezm"
   },
   "source": [
    "Print what the best action in each state is."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "K18q4KqpybaZ",
    "outputId": "b71c2f87-b883-4875-8184-ff55b7f99c06"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[0, 3, 3, 2, 0, 0, 2, 2, 0, 2, 2, 2, 1, 1, 1, 0]\n"
     ]
    }
   ],
   "source": [
    "my_value_iterator.print_best_actions()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "v6a4e8bNQezs",
    "scrolled": true
   },
   "source": [
    "Print value in each state is."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "su6QHrlDQezt",
    "outputId": "9351924a-6ccd-4205-f619-3e908d4f4010"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[ 0.   -1.25 -2.5  -3.75 -1.53 -2.63 -3.14 -2.5  -2.82 -3.01 -2.05 -1.25\n -3.   -2.   -1.    0.  ]\n"
     ]
    }
   ],
   "source": [
    "my_value_iterator.print_states_value()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YeNzWkMNQezw"
   },
   "source": [
    "Find the best policy using the Policy Iteration algorithm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "iWi4twcEw-aw"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class PolicyIteration(object):\n",
    "    def __init__(self, env, discount_factor, theta):\n",
    "        self.env = env\n",
    "        self.discount_factor = discount_factor\n",
    "        self.theta = theta\n",
    "        self.states_value = np.zeros(self.env.nS)\n",
    "        self.policy = np.ones([self.env.nS, self.env.nA]) / self.env.nA\n",
    "        self.action_values = np.zeros(self.env.nA)\n",
    "        self.best_actions = []\n",
    "\n",
    "    def find_action_value(self, curr_state):\n",
    "        states_value = 0\n",
    "        for action, action_probability in enumerate(self.policy[curr_state]):\n",
    "            for p_s_ns, next_state, state_reward, done in env.P[curr_state][action]:\n",
    "                states_value += action_probability * p_s_ns * (state_reward + discount_factor * self.states_value[next_state])\n",
    "        return states_value\n",
    "                \n",
    "    def find_best_action_value(self, curr_state):\n",
    "        self.action_values = np.zeros(self.env.nA)\n",
    "        self.policy_eval(self.policy)\n",
    "        for action in range(env.nA):\n",
    "            for p_s_ns, next_state, state_reward, done in env.P[curr_state][action]:\n",
    "                self.action_values[action] += p_s_ns * (state_reward + discount_factor * self.states_value[next_state])\n",
    "        return np.argmax(self.action_values)\n",
    "    \n",
    "    def find_best_action(self, curr_state):\n",
    "        states_value = self.find_action_value(curr_state)\n",
    "        best_action = np.argmax(states_value)\n",
    "        self.best_actions.append(best_action)\n",
    "        return best_action\n",
    "\n",
    "    def policy_eval(self,policy):\n",
    "        while True:\n",
    "            delta = 0\n",
    "            for state in range(self.env.nS):\n",
    "                v = 0\n",
    "                v_temp = self.states_value[state]\n",
    "                v = self.find_action_value(state)\n",
    "                delta = max(delta, np.abs(v_temp - v))\n",
    "                self.states_value[state] = v\n",
    "            if delta < self.theta:\n",
    "                    break\n",
    "        return self.states_value\n",
    "    \n",
    "    def find_policy(self):\n",
    "        best_action = self.find_best_action(state)\n",
    "        self.policy[state, best_action] = 1.0\n",
    "        print(self.policy)\n",
    "        \n",
    "    def policy_improvment(self):\n",
    "        while True:\n",
    "            self.policy_eval(self.policy)\n",
    "            policy_stable = True\n",
    "            for state in range(self.env.nS):\n",
    "                chosen_action = np.argmax(self.policy[state])\n",
    "                best_action = self.find_best_action_value(state)\n",
    "                if  chosen_action != best_action:\n",
    "                    policy_stable = False\n",
    "                self.policy[state] = np.eye(env.nA)[best_action]\n",
    "                self.best_actions.append(best_action)\n",
    "            if policy_stable : return self.policy \n",
    "\n",
    "\n",
    "    def print_best_actions(self):\n",
    "        print((np.array(self.best_actions)[:-1]).reshape((4,16)))\n",
    "\n",
    "    def print_states_value(self):\n",
    "        print(self.states_value)\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "98520w11Qez0"
   },
   "source": [
    "policy_improvement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "VU4atA6N80fd",
    "outputId": "b5ea3889-4668-4171-a832-0baffbecd4d3"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[[1. 0. 0. 0.]\n [0. 0. 0. 1.]\n [0. 0. 0. 1.]\n [0. 0. 1. 0.]\n [1. 0. 0. 0.]\n [1. 0. 0. 0.]\n [0. 0. 1. 0.]\n [0. 0. 1. 0.]\n [1. 0. 0. 0.]\n [0. 0. 1. 0.]\n [0. 0. 1. 0.]\n [0. 0. 1. 0.]\n [0. 1. 0. 0.]\n [0. 1. 0. 0.]\n [0. 1. 0. 0.]\n [1. 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "discount_factor=1.0\n",
    "theta=0.00001\n",
    "my_policy_iterator = PolicyIteration(env, discount_factor, theta)\n",
    "\n",
    "my_policy_iterator.policy_improvment()\n",
    "my_policy_iterator.find_policy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uztiJNQ_Qez7"
   },
   "source": [
    "Print what the best action in each state is."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "V9qqUA_nQez8",
    "outputId": "66f77a4d-ff03-4164-80b0-c81479cebd16"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[[0 3 3 3 0 0 0 0 0 0 0 2 0 1 1 0]\n [0 3 3 3 0 0 0 2 0 2 2 2 1 1 1 0]\n [0 3 3 2 0 0 2 2 0 2 2 2 1 1 1 0]\n [0 3 3 2 0 0 2 2 0 2 2 2 1 1 1 0]]\n"
     ]
    }
   ],
   "source": [
    "my_policy_iterator.print_best_actions()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-vViqFHOQez_"
   },
   "source": [
    "Print value in each state is."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "xOEc0ADxQe0A",
    "outputId": "00ce22e3-e3cc-4afd-8250-9a7282ad54ad"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[ 0.      -1.25    -2.5     -3.75    -1.5256  -2.628   -3.14    -2.5\n -2.82248 -3.01    -2.05    -1.25    -3.      -2.      -1.       0.     ]\n"
     ]
    }
   ],
   "source": [
    "my_policy_iterator.print_states_value()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "7JlwUgDs2HpV"
   },
   "outputs": [],
   "source": [
    "def value_iteration(env, theta=0.0001, discount_factor=1.0):\n",
    "    \"\"\"\n",
    "    Value Iteration Algorithm.\n",
    "    \n",
    "    Args:\n",
    "        env: OpenAI env. env.P represents the transition probabilities of the environment.\n",
    "            env.P[s][a] is a list of transition tuples (prob, next_state, reward, done).\n",
    "            env.nS is a number of states in the environment. \n",
    "            env.nA is a number of actions in the environment.\n",
    "        theta: We stop evaluation once our value function change is less than theta for all states.\n",
    "        discount_factor: Gamma discount factor.\n",
    "        \n",
    "    Returns:\n",
    "        A tuple (policy, V) of the optimal policy and the optimal value function.        \n",
    "    \"\"\"\n",
    "\n",
    "    V = np.zeros(env.nS)\n",
    "    policy = np.zeros([env.nS, env.nA])\n",
    "\n",
    "    while True:\n",
    "      break\n",
    "    return policy, V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "6wg8-n0S192g"
   },
   "outputs": [],
   "source": [
    "def policy_eval(policy, env, discount_factor=1.0, theta=0.00001):\n",
    "    \"\"\"\n",
    "    Evaluate a policy given an environment and a full description of the environment's dynamics.\n",
    "    \n",
    "    Args:\n",
    "        policy: [S, A] shaped matrix representing the policy.\n",
    "        env: OpenAI env. env.P represents the transition probabilities of the environment.\n",
    "            env.P[s][a] is a list of transition tuples (prob, next_state, reward, done).\n",
    "            env.nS is a number of states in the environment. \n",
    "            env.nA is a number of actions in the environment.\n",
    "        theta: We stop evaluation once our value function change is less than theta for all states.\n",
    "        discount_factor: Gamma discount factor.\n",
    "    \n",
    "    Returns:\n",
    "        Vector of length env.nS representing the value function.\n",
    "    \"\"\"\n",
    "    # Start with a random (all 0) value function\n",
    "    V = np.zeros(env.nS)\n",
    "    V_pre = np.zeros(env.nS)\n",
    "    while True:\n",
    "        break\n",
    "    return np.array(V)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "akyZXgjf1-Jd"
   },
   "outputs": [],
   "source": [
    "def policy_improvement(env, policy_eval_fn = policy_eval, discount_factor=1.0):\n",
    "    \"\"\"\n",
    "    Policy Improvement Algorithm. Iteratively evaluates and improves a policy\n",
    "    until an optimal policy is found.\n",
    "            \n",
    "    Args:\n",
    "        env: The OpenAI environment.\n",
    "        policy_eval_fn: Policy Evaluation function that takes 3 arguments:\n",
    "            policy, env, discount_factor.\n",
    "        discount_factor: gamma discount factor.\n",
    "        \n",
    "    Returns:\n",
    "        A tuple (policy, V). \n",
    "        policy is the optimal policy, a matrix of shape [S, A] where each state s\n",
    "        contains a valid probability distribution over actions.\n",
    "        V is the value function for the optimal policy.\n",
    "        \n",
    "    \"\"\"\n",
    "\n",
    "    def one_step_lookahead(state, V):\n",
    "        \"\"\"\n",
    "        Helper function to calculate the value for all action in a given state.\n",
    "        \n",
    "        Args:\n",
    "            state: The state to consider (int)\n",
    "            V: The value to use as an estimator, Vector of length env.nS\n",
    "        \n",
    "        Returns:\n",
    "            A vector of length env.nA containing the expected value of each action.\n",
    "        \"\"\"\n",
    "    # Start with a random policy\n",
    "    policy = np.ones([env.nS, env.nA]) / env.nA\n",
    "    while True:\n",
    "      break  \n",
    "\n",
    "    \n",
    "    return policy, V"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "HandsOn_1.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}