Experiment-2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Final version - Bairds \"7-star\" counter-example\n",
    "import numpy as np\n",
    "from scipy.linalg import eig \n",
    "import mdptoolbox, mdptoolbox.example\n",
    "import random\n",
    "from numpy.random import seed\n",
    "import time\n",
    "\n",
    "\n",
    "def get_next_state(state,action):\n",
    "    \n",
    "    if action == 0:\n",
    "        next_state = 6\n",
    "    else:\n",
    "        next_state = np.random.choice(6)\n",
    "    \n",
    "    return next_state\n",
    "    \n",
    "    \n",
    "def feature_state(thetha,state):\n",
    "    if state == 0:\n",
    "        return 2*thetha[0] + thetha[7]\n",
    "    if state == 1:\n",
    "        return 2*thetha[1]+thetha[7]\n",
    "    if state == 2:\n",
    "        return 2*thetha[2]+thetha[7]\n",
    "    if state == 3:\n",
    "        return 2*thetha[3]+thetha[7]\n",
    "    if state == 4:\n",
    "        return 2*thetha[4]+thetha[7]\n",
    "    if state == 5:\n",
    "        return 2*thetha[5]+thetha[7]\n",
    "    if state == 6:\n",
    "        return thetha[6] + 2*thetha[7]\n",
    "\n",
    "\n",
    "\n",
    "def grad_state(state):\n",
    "    if state == 0:\n",
    "        return np.array([2.0,0.0,0,0,0,0,0,1])\n",
    "    if state == 1:\n",
    "        return np.array([0.0,2.0,0,0,0,0,0,1])\n",
    "    if state == 2:\n",
    "        return np.array([0.0,0.0,2,0,0,0,0,1])\n",
    "    if state == 3:\n",
    "        return np.array([0.0,0.0,0,2,0,0,0,1])\n",
    "    if state == 4:\n",
    "        return np.array([0.0,0.0,0,0,2,0,0,1])\n",
    "    if state == 5:\n",
    "        return np.array([0.0,0.0,0,0,0,2,0,1])\n",
    "    if state == 6:\n",
    "        return np.array([0.0,0.0,0,0,0,0,1,2])\n",
    "    \n",
    "\n",
    "total_runs = 10\n",
    "total_iterations = 1000000\n",
    "\n",
    "step_size = 0.0001\n",
    "\n",
    "error_storage_td = np.zeros((total_runs,total_iterations))\n",
    "error_storage_ptd = np.zeros((total_runs,total_iterations))\n",
    "error_storage_etd = np.zeros((total_runs,total_iterations))\n",
    "\n",
    "\n",
    "for runs in range(total_runs):\n",
    "    \n",
    "    np.random.seed((runs+1)*100)\n",
    "    random.seed((runs+1)*110)\n",
    "    \n",
    "    opt = np.array([[0,0,0,0,0,0,0]])\n",
    "\n",
    "    \n",
    "    thetha = [1,1,1,1,1,1,10,1]\n",
    "    thetha_td = [1,1,1,1,1,1,10,1]\n",
    "    discount = 0.99\n",
    "    \n",
    "    \n",
    "    pi = np.array([[1,0],[1,0],[1,0],[1,0],[1,0],[1,0],[1,0]])\n",
    "    mu = np.array([[1/7,6/7],[1/7,6/7],[1/7,6/7],[1/7,6/7],[1/7,6/7],[1/7,6/7],[1/7,6/7]])\n",
    "        \n",
    "    d_mu = np.array([[1/7,1/7,1/7,1/7,1/7,1/7,1/7]])\n",
    "        \n",
    "    eta = 1\n",
    "    \n",
    "    F=1\n",
    "    thetha_ETD = [1,1,1,1,1,1,10,1]\n",
    "    state = np.random.choice(7)\n",
    "    for main_iter in range(total_iterations):\n",
    "        \n",
    "        action = np.random.choice(2,1,p=[1/7,6/7])[0]\n",
    "        importance_sampling = pi[state,action]/mu[state,action]     \n",
    "        next_state = get_next_state(state,action)\n",
    "\n",
    "\n",
    "        grad = grad_state(state)\n",
    "        \n",
    "        delta = 0 + discount*feature_state(thetha_ETD,next_state) - feature_state(thetha_ETD,state)\n",
    "        temp = F*importance_sampling*step_size*delta\n",
    "        thetha_ETD = thetha_ETD + temp*grad\n",
    "\n",
    "        delta = 0 + discount*feature_state(thetha,next_state) - (1+eta)*feature_state(thetha,state)\n",
    "        temp2 = importance_sampling*step_size*delta\n",
    "        thetha = thetha + temp2*grad\n",
    "        \n",
    "        delta = 0 + discount*feature_state(thetha_td,next_state) - feature_state(thetha_td,state)\n",
    "        temp3 = importance_sampling*step_size*delta\n",
    "        thetha_td = thetha_td + temp3*grad\n",
    "        \n",
    "\n",
    "        F = 1+ discount*importance_sampling*F \n",
    "        state = next_state\n",
    "        \n",
    "        v_ptd = np.zeros((7,1))\n",
    "        v_etd = np.zeros((7,1))\n",
    "        v_td =  np.zeros((7,1))\n",
    "        \n",
    "        for i in range(7):\n",
    "            v_ptd[i,0] = feature_state(thetha,i)\n",
    "            v_etd[i,0] = feature_state(thetha_ETD,i)\n",
    "            v_td[i,0] = feature_state(thetha_td,i)\n",
    "        \n",
    "        error_storage_ptd[runs,main_iter] = np.sqrt(np.sum(np.multiply(d_mu.T,np.square(opt.T - v_ptd))))\n",
    "        error_storage_etd[runs,main_iter] = np.sqrt(np.sum(np.multiply(d_mu.T,np.square(opt.T - v_etd))))\n",
    "        error_storage_td[runs,main_iter] = np.sqrt(np.sum(np.multiply(d_mu.T,np.square(opt.T - v_td))))\n",
    "        \n",
    "\n",
    "        \n",
    "print(thetha,thetha_ETD)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}