DRL_10_VI_Algorithm_for_V.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "DRL_10_Dinamic Programming.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "suvKdcFqXcd5"
      },
      "source": [
        "DEEP REINFORCEMENT LEARNING EXPLAINED - 10\n",
        "# **Dynamic Programming**\n",
        "## V-function in Practice for Frozen-Lake Environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fLiVqPvwmCJB"
      },
      "source": [
        "import gym\n",
        "import collections\n",
        "from torch.utils.tensorboard import SummaryWriter\n",
        "import numpy as np\n",
        "\n",
        "ENV_NAME = \"FrozenLake-v0\"\n",
        "#ENV_NAME = \"FrozenLake8x8-v0\"  \n",
        "GAMMA = 0.95\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cxTWfToPrjxe"
      },
      "source": [
        "### The Agent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CgnZfMGvk9Rl"
      },
      "source": [
        "\n",
        "class Agent:\n",
        "    def __init__(self):\n",
        "        self.env = gym.make(ENV_NAME)\n",
        "        self.V = np.zeros(self.env.observation_space.n)\n",
        "\n",
        "    def calc_action_value(self, state, action):\n",
        "        action_value = sum([prob*(r + GAMMA * self.V[s_])\n",
        "                          for prob, s_, r, _ in self.env.P[state][action]]) \n",
        "        return action_value\n",
        "\n",
        "    def select_action(self, state):\n",
        "        best_action, best_value = None, None\n",
        "        for action in range(self.env.action_space.n):\n",
        "            action_value = self.calc_action_value(state, action)\n",
        "            if best_value is None or best_value < action_value:\n",
        "                best_value = action_value\n",
        "                best_action = action\n",
        "        return best_action\n",
        "\n",
        "\n",
        "    def value_iteration(self):\n",
        "        for state in range(self.env.observation_space.n):\n",
        "            state_values = [self.calc_action_value(state, action)\n",
        "                           for action in range(self.env.action_space.n)\n",
        "                           ]\n",
        "            self.V[state] = max(state_values)\n",
        "        return self.V"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "scTw9juUrf3u"
      },
      "source": [
        "### Training loop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EcUs5zNa7vUI"
      },
      "source": [
        "TEST_EPISODES = 40\n",
        "REWARD_GOAL = 0.90\n",
        "\n",
        "def train(agent): \n",
        "  test_env = gym.make(ENV_NAME)\n",
        "  writer = SummaryWriter()\n",
        "\n",
        "  iter_no = 0\n",
        "  best_reward = 0.0\n",
        " \n",
        "  while best_reward < REWARD_GOAL:\n",
        "\n",
        "        #step 1\n",
        "        agent.value_iteration()\n",
        "\n",
        "        #step 2 check the improvements \n",
        "        iter_no += 1\n",
        "        reward_test = 0.0\n",
        "        for _ in range(TEST_EPISODES):\n",
        "            total_reward = 0.0\n",
        "            state = test_env.reset()\n",
        "            while True:\n",
        "                action = agent.select_action(state)\n",
        "                new_state, new_reward, is_done, _ = test_env.step(action)\n",
        "                total_reward += new_reward\n",
        "                if is_done: break\n",
        "                state = new_state\n",
        "            reward_test += total_reward\n",
        "        reward_test /= TEST_EPISODES\n",
        "\n",
        "        #step track with TensorBoard \n",
        "        writer.add_scalar(\"reward\", reward_test, iter_no)\n",
        "        if reward_test > best_reward:\n",
        "            print(\"Best reward updated %.2f at iteration %d \" % (reward_test ,iter_no) )\n",
        "            best_reward = reward_test\n",
        "\n",
        "  writer.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I_yLS-RjBVjl"
      },
      "source": [
        "### Training the Agent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5-1Z8BhCjk0f",
        "outputId": "f621fe9f-e13d-4fb6-9df7-17d2f4d98421"
      },
      "source": [
        "agent = Agent()\n",
        "train(agent)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Best reward updated 0.28 at iteration 3 \n",
            "Best reward updated 0.35 at iteration 6 \n",
            "Best reward updated 0.45 at iteration 7 \n",
            "Best reward updated 0.55 at iteration 8 \n",
            "Best reward updated 0.75 at iteration 9 \n",
            "Best reward updated 0.85 at iteration 18 \n",
            "Best reward updated 0.88 at iteration 57 \n",
            "Best reward updated 0.90 at iteration 78 \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "phD7ZXEU1JEL"
      },
      "source": [
        "def extract_policy(agent):   \n",
        "    env = gym.make(ENV_NAME)\n",
        "    policy = np.zeros(env.observation_space.n) \n",
        "    for s in range(env.observation_space.n):\n",
        "        Q_values = [agent.calc_action_value(s,a) for a in range(env.action_space.n)] \n",
        "        policy[s] = np.argmax(np.array(Q_values))        \n",
        "    return policy\n",
        "\n",
        "def print_policy(policy):\n",
        "    print(policy.reshape([-1, 4]))\n",
        "    print(\"\\n\")\n",
        "    a2w = {0:'<', 1:'v', 2:'>', 3:'^'}\n",
        "    policy_arrows = [a2w[x] for x in policy]\n",
        "    print(np.array(policy_arrows).reshape([-1, 4]))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qE-evSt6tnGi",
        "outputId": "e1f782ed-01b4-4176-efbf-e06d4869ea08"
      },
      "source": [
        "policy=extract_policy(agent)\n",
        "print_policy(policy)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0. 3. 0. 3.]\n",
            " [0. 0. 0. 0.]\n",
            " [3. 1. 0. 0.]\n",
            " [0. 2. 1. 0.]]\n",
            "\n",
            "\n",
            "[['<' '^' '<' '^']\n",
            " ['<' '<' '<' '<']\n",
            " ['^' 'v' '<' '<']\n",
            " ['<' '>' 'v' '<']]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ntbG1_bJXcJ8",
        "outputId": "d7923d22-b0b3-418a-e71c-5aaee7155057"
      },
      "source": [
        "env = gym.make(ENV_NAME)\n",
        "env.env.P[0][0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(0.3333333333333333, 0, 0.0, False),\n",
              " (0.3333333333333333, 0, 0.0, False),\n",
              " (0.3333333333333333, 4, 0.0, False)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4avOvWPjXigX",
        "outputId": "bc363ba1-12f4-424a-8682-3f69c609633e"
      },
      "source": [
        "env.env.P[4][0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(0.3333333333333333, 0, 0.0, False),\n",
              " (0.3333333333333333, 4, 0.0, False),\n",
              " (0.3333333333333333, 8, 0.0, False)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ipzqA0qpclTy",
        "outputId": "81f1d87e-0ae0-4c9f-b093-618ce74e4900"
      },
      "source": [
        "env.env.P[8][3]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(0.3333333333333333, 9, 0.0, False),\n",
              " (0.3333333333333333, 4, 0.0, False),\n",
              " (0.3333333333333333, 8, 0.0, False)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PHUCg19lBafh"
      },
      "source": [
        "### Test the Agent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YPjUCudxM8S1"
      },
      "source": [
        "def test(agent):\n",
        "    new_test_env = gym.make(ENV_NAME) \n",
        "    state = new_test_env.reset()\n",
        "    new_test_env.render()\n",
        "    is_done = False\n",
        "    iter_no = 0\n",
        "\n",
        "    while not is_done:\n",
        "        action = agent.select_action(state)\n",
        "        new_state, reward, is_done, _ = new_test_env.step(action)\n",
        "        new_test_env.render()\n",
        "        state = new_state\n",
        "        iter_no +=1\n",
        "    print(\"reward =    \", reward)\n",
        "    print(\"iterations =\", iter_no)\n",
        "\n",
        "test(agent)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ymICwXwAkyuC"
      },
      "source": [
        "%load_ext tensorboard\n"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YmobnomGlHBt"
      },
      "source": [
        "tensorboard  --logdir=runs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wl0BbyRJ-M_x"
      },
      "source": [
        "### The Agent that estimates the Transition Function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z1eGAk3_-KTp"
      },
      "source": [
        "N =100\n",
        "class Agent_Updated:\n",
        "    def __init__(self):\n",
        "        self.env = gym.make(ENV_NAME)\n",
        "        self.state = self.env.reset()\n",
        "        self.rewards = collections.defaultdict(float)\n",
        "        self.transits = collections.defaultdict(\n",
        "            collections.Counter)\n",
        "        self.V = np.zeros(self.env.observation_space.n)\n",
        "\n",
        "    def play_n_random_steps(self, count):\n",
        "        for _ in range(count):\n",
        "            action = self.env.action_space.sample()\n",
        "            new_state, reward, is_done, _ = self.env.step(action)\n",
        "            self.rewards[(self.state, action, new_state)] = reward\n",
        "            self.transits[(self.state, action)][new_state] += 1\n",
        "            if is_done:\n",
        "                self.state = self.env.reset() \n",
        "            else: \n",
        "                self.state = new_state\n",
        "\n",
        "    def calc_action_value(self, state, action):\n",
        "        target_counts = self.transits[(state, action)]\n",
        "        total = sum(target_counts.values())\n",
        "        action_value = 0.0\n",
        "        for s_, count in target_counts.items():\n",
        "            r = self.rewards[(state, action, s_)]\n",
        "            prob = (count / total)\n",
        "            action_value += prob*(r + GAMMA * self.V[s_])\n",
        "        return action_value\n",
        "\n",
        "    def select_action(self, state):\n",
        "        best_action, best_value = None, None\n",
        "        for action in range(self.env.action_space.n):\n",
        "            action_value = self.calc_action_value(state, action)\n",
        "            if best_value is None or best_value < action_value:\n",
        "                best_value = action_value\n",
        "                best_action = action\n",
        "        return best_action\n",
        "\n",
        "\n",
        "    def value_iteration(self):\n",
        "        self.play_n_random_steps(N)\n",
        "        for state in range(self.env.observation_space.n):\n",
        "            state_values = [\n",
        "                self.calc_action_value(state, action)\n",
        "                for action in range(self.env.action_space.n)\n",
        "            ]\n",
        "            self.V[state] = max(state_values)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6-s8R6r-CG_2",
        "outputId": "4113966f-adbb-49ba-f078-e0100932f7c8"
      },
      "source": [
        "agent = Agent_Updated()\n",
        "train(agent)\n",
        "policy=extract_policy(agent)\n",
        "print_policy(policy)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Best reward updated 0.07 at iteration 5 \n",
            "Best reward updated 0.12 at iteration 7 \n",
            "Best reward updated 0.30 at iteration 8 \n",
            "Best reward updated 0.33 at iteration 12 \n",
            "Best reward updated 0.42 at iteration 13 \n",
            "Best reward updated 0.60 at iteration 42 \n",
            "Best reward updated 0.70 at iteration 56 \n",
            "Best reward updated 0.75 at iteration 66 \n",
            "Best reward updated 0.80 at iteration 70 \n",
            "Best reward updated 0.82 at iteration 78 \n",
            "Best reward updated 0.85 at iteration 80 \n",
            "Best reward updated 0.88 at iteration 98 \n",
            "Best reward updated 0.93 at iteration 528 \n",
            "[[0. 3. 0. 3.]\n",
            " [0. 0. 2. 0.]\n",
            " [3. 1. 0. 0.]\n",
            " [0. 2. 1. 0.]]\n",
            "\n",
            "\n",
            "[['<' '^' '<' '^']\n",
            " ['<' '<' '>' '<']\n",
            " ['^' 'v' '<' '<']\n",
            " ['<' '>' 'v' '<']]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-djx4eMWDWWR"
      },
      "source": [
        "tensorboard  --logdir=runs"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}