11_DQN_Reinforcement_learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 405 DQN Reinforcement Learning\n",
    "\n",
    "View more, visit my tutorial page: https://morvanzhou.github.io/tutorials/\n",
    "My Youtube Channel: https://www.youtube.com/user/MorvanZhou\n",
    "More about Reinforcement learning: https://morvanzhou.github.io/tutorials/machine-learning/reinforcement-learning/\n",
    "\n",
    "Dependencies:\n",
    "* torch: 0.1.11\n",
    "* gym: 0.8.1\n",
    "* numpy\n",
    "\n",
    "强化学习\n",
    "\n",
    "具有记忆能力\n",
    "\n",
    "一个\n",
    "\n",
    "## 程序框架\n",
    "### 1. 定义两个class\n",
    "- Net 神经网络\n",
    "    - def __init__(self, ):\n",
    "    - def forward(self, x):\n",
    "- DQN\n",
    "    - def __init__(self):\n",
    "    - def choose_action(self, x):根据环境中的观测值，采取相应动作\n",
    "    - def store_transition(self, s, a, r, s_):记忆库_存储之前的记忆\n",
    "    - def learn(self):学习记忆库中的记忆\n",
    "    \n",
    "### 2. 实施步骤\n",
    "```\n",
    "for i_episode in range(400):\n",
    "    s = env.reset()# status 跟环境互动，得到当前状态\n",
    "    while True:\n",
    "        env.render()# 环境渲染\n",
    "        \n",
    "        a = dqn.choose_action(s)# DQN根据状态s计算出相应的行为\n",
    "        s_, r, done, info = env.step(a)# 环境根据行为给出新的反馈\n",
    "        dqn.store_transition(s, a, r, s_)# 存储状态、动作、反馈，进行下一步\n",
    "        if dqn.memory_counter > MEMORY_CAPACITY:# DQN学习\n",
    "        if done:\n",
    "            break\n",
    "        s = s_\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# Hyper Parameters\n",
    "BATCH_SIZE = 32\n",
    "LR = 0.01                   # learning rate\n",
    "EPSILON = 0.9               # greedy policy\n",
    "GAMMA = 0.9                 # reward discount\n",
    "TARGET_REPLACE_ITER = 100   # target update frequency\n",
    "MEMORY_CAPACITY = 2000\n",
    "env = gym.make('CartPole-v0')# 导入小车实验\n",
    "env = env.unwrapped\n",
    "N_ACTIONS = env.action_space.n\n",
    "N_STATES = env.observation_space.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, ):\n",
    "        super(Net, self).__init__()\n",
    "        self.fc1 = nn.Linear(N_STATES, 10)\n",
    "        self.fc1.weight.data.normal_(0, 0.1)   # initialization\n",
    "        self.out = nn.Linear(10, N_ACTIONS)\n",
    "        self.out.weight.data.normal_(0, 0.1)   # initialization\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.fc1(x)\n",
    "        x = F.relu(x)\n",
    "        actions_value = self.out(x)\n",
    "        return actions_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class DQN(object):\n",
    "    def __init__(self):\n",
    "        self.eval_net, self.target_net = Net(), Net()# eval重新计算\n",
    "        # 会把eval转换到target，达到延迟更新的效果\n",
    "\n",
    "        self.learn_step_counter = 0                                     # for target updating\n",
    "        self.memory_counter = 0                                         # for storing memory\n",
    "        self.memory = np.zeros((MEMORY_CAPACITY, N_STATES * 2 + 2))     # initialize memory\n",
    "        self.optimizer = torch.optim.Adam(self.eval_net.parameters(), lr=LR)\n",
    "        self.loss_func = nn.MSELoss()\n",
    "\n",
    "    def choose_action(self, x):\n",
    "        x = Variable(torch.unsqueeze(torch.FloatTensor(x), 0))# X是观测值\n",
    "        # input only one sample\n",
    "        if np.random.uniform() < EPSILON:   # greedy if下 采取两种动作\n",
    "            actions_value = self.eval_net.forward(x)\n",
    "            action = torch.max(actions_value, 1)[1].data.numpy()[0, 0]     # return the argmax\n",
    "        else:   # random\n",
    "            action = np.random.randint(0, N_ACTIONS)\n",
    "        return action\n",
    "\n",
    "    def store_transition(self, s, a, r, s_):\n",
    "        transition = np.hstack((s, [a, r], s_))\n",
    "        # replace the old memory with new memory\n",
    "        index = self.memory_counter % MEMORY_CAPACITY\n",
    "        self.memory[index, :] = transition\n",
    "        self.memory_counter += 1# 覆盖老的记忆\n",
    "\n",
    "    def learn(self):\n",
    "        # target parameter update 老的记忆学习\n",
    "        if self.learn_step_counter % TARGET_REPLACE_ITER == 0:\n",
    "            self.target_net.load_state_dict(self.eval_net.state_dict())\n",
    "        self.learn_step_counter += 1\n",
    "\n",
    "        # sample batch transitions\n",
    "        sample_index = np.random.choice(MEMORY_CAPACITY, BATCH_SIZE)\n",
    "        b_memory = self.memory[sample_index, :]\n",
    "        b_s = Variable(torch.FloatTensor(b_memory[:, :N_STATES]))\n",
    "        b_a = Variable(torch.LongTensor(b_memory[:, N_STATES:N_STATES+1].astype(int)))\n",
    "        b_r = Variable(torch.FloatTensor(b_memory[:, N_STATES+1:N_STATES+2]))\n",
    "        b_s_ = Variable(torch.FloatTensor(b_memory[:, -N_STATES:]))\n",
    "\n",
    "        # q_eval w.r.t the action in experience\n",
    "        q_eval = self.eval_net(b_s).gather(1, b_a)  # shape (batch, 1)\n",
    "        q_next = self.target_net(b_s_).detach()     # detach from graph, don't backpropagate\n",
    "        q_target = b_r + GAMMA * q_next.max(1)[0]   # shape (batch, 1)\n",
    "        loss = self.loss_func(q_eval, q_target)\n",
    "\n",
    "        self.optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        self.optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dqn = DQN()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Collecting experience...\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "too many indices for array",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-70a490e86ec3>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      6\u001b[0m     \u001b[1;32mwhile\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m         \u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m         \u001b[0ma\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdqn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mchoose_action\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      9\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m         \u001b[1;31m# take action\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-4-615b730a722e>\u001b[0m in \u001b[0;36mchoose_action\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m     14\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0muniform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m<\u001b[0m \u001b[0mEPSILON\u001b[0m\u001b[1;33m:\u001b[0m   \u001b[1;31m# greedy\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m             \u001b[0mactions_value\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0meval_net\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 16\u001b[1;33m             \u001b[0maction\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mactions_value\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m     \u001b[1;31m# return the argmax\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     17\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m   \u001b[1;31m# random\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     18\u001b[0m             \u001b[0maction\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrandint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_ACTIONS\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mIndexError\u001b[0m: too many indices for array"
     ]
    }
   ],
   "source": [
    "\n",
    "print('\\nCollecting experience...')\n",
    "for i_episode in range(400):\n",
    "    s = env.reset()\n",
    "    ep_r = 0\n",
    "    while True:\n",
    "        env.render()\n",
    "        a = dqn.choose_action(s)\n",
    "\n",
    "        # take action\n",
    "        s_, r, done, info = env.step(a)\n",
    "\n",
    "        # modify the reward\n",
    "        x, x_dot, theta, theta_dot = s_\n",
    "        r1 = (env.x_threshold - abs(x)) / env.x_threshold - 0.8\n",
    "        r2 = (env.theta_threshold_radians - abs(theta)) / env.theta_threshold_radians - 0.5\n",
    "        r = r1 + r2\n",
    "\n",
    "        dqn.store_transition(s, a, r, s_)\n",
    "\n",
    "        ep_r += r\n",
    "        if dqn.memory_counter > MEMORY_CAPACITY:\n",
    "            dqn.learn()\n",
    "            if done:\n",
    "                print('Ep: ', i_episode,\n",
    "                      '| Ep_r: ', round(ep_r, 2))\n",
    "\n",
    "        if done:\n",
    "            break\n",
    "        s = s_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}