DQN Training.ipynb

{
 "metadata": {
  "language": "lua",
  "name": "",
  "signature": "sha256:26d053e2027fa00b44d057e52f3d127d412207bdb38750c7d6d1f794982739d7"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# Deep Q-Network Training\n",
      "\n",
      "In this worksheet we are going to go over training a Reinforcement Learning (RL) agent trained on ATARI 2600 games. We will use the famous game of pong as a our training test bed.\n",
      "\n",
      "One can download the code required to train the DQN agent at https://sites.google.com/a/deepmind.com/dqn/"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Setup\n",
      "We first start by setting up a function to update our lua path so that DQN training code can be loaded."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "--\n",
      "-- Most probably this function should have been part of statndard lua or Torch\n",
      "--\n",
      "function addpath(new_path)\n",
      "    package.path = package.path .. ';' .. new_path .. '/?.lua;' .. new_path .. '/?/init.lua'\n",
      "end\n",
      "--\n",
      "-- This is where the code lives on your machine.\n",
      "--\n",
      "addpath('/home/ubuntu/DQN/Human_Level_Control_through_Deep_Reinforcement_Learning/dqn/')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "--\n",
      "-- The training arguments are stored in a file in the following file. We load them here and then explain one by one.\n",
      "-- In lua, when a process is run a global 'arg' table contains all the command line arguments, just like **argv.\n",
      "--\n",
      "arg = torch.load('/home/ubuntu/DQN/Human_Level_Control_through_Deep_Reinforcement_Learning/dqn/pong_arguments.t7')\n",
      "print(arg)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 2,
       "text": [
        "{\n",
        "  0 : train_agent.lua\n",
        "  1 : -framework\n",
        "  2 : alewrap\n",
        "  3 : -game_path\n",
        "  4 : /home/ubuntu/DQN/Human_Level_Control_through_Deep_Reinforcement_Learning/roms/\n",
        "  5 : -name\n",
        "  6 : DQN3_0_1_pong_FULL_Y\n",
        "  7 : -env\n",
        "  8 : pong\n",
        "  9 : -env_params\n",
        "  10 : useRGB=true\n",
        "  11 : -agent\n",
        "  12 : NeuralQLearner\n",
        "  13 : -agent_params\n",
        "  14 : lr=0.00025,ep=1,ep_end=0.1,ep_endt=replay_memory,discount=0.99,hist_len=4,learn_start=50000,replay_memory=1000000,update_freq=4,n_replay=1,network=\"convnet_atari3\",preproc=\"net_downsample_2x_full_y\",state_dim=7056,minibatch_size=32,rescale_r=1,ncols=1,bufferSize=512,valid_size=500,target_q=10000,clip_delta=1,min_reward=-1,max_reward=1\n",
        "  15 : -steps\n",
        "  16 : 50000000\n",
        "  17 : -eval_freq\n",
        "  18 : 250000\n",
        "  19 : -eval_steps\n",
        "  20 : 125000\n",
        "  21 : -prog_freq\n",
        "  22 : 10000\n",
        "  23 : -save_freq\n",
        "  24 : 125000\n",
        "  25 : -actrep\n",
        "  26 : 4\n",
        "  27 : -gpu\n",
        "  28 : 0\n",
        "  29 : -random_starts\n",
        "  30 : 30\n",
        "  31 : -pool_frms\n",
        "  32 : type=\"max\",size=2\n",
        "  33 : -seed\n",
        "  34 : 1\n",
        "  35 : -threads\n",
        "  36 : 4\n",
        "  -2 : -l\n",
        "  -1 : env\n",
        "  -3 : /home/ubuntu/torch/install/bin/luajit\n",
        "}\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "--\n",
      "-- Load initialization functions\n",
      "--\n",
      "require \"initenv\";"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "--\n",
      "-- We start by defining the valid command line arguments. Each argument is defined by its flag, followed by its\n",
      "-- default value and finally a help string.\n",
      "--\n",
      "cmd = torch.CmdLine()\n",
      "cmd:text()\n",
      "cmd:text('Train Agent in Environment:')\n",
      "cmd:text()\n",
      "cmd:text('Options:')\n",
      "\n",
      "cmd:option('-framework', '', 'name of training framework')\n",
      "cmd:option('-env', '', 'name of environment to use')\n",
      "cmd:option('-game_path', '', 'path to environment file (ROM)')\n",
      "cmd:option('-env_params', '', 'string of environment parameters')\n",
      "cmd:option('-pool_frms', '',\n",
      "           'string of frame pooling parameters (e.g.: size=2,type=\"max\")')\n",
      "cmd:option('-actrep', 1, 'how many times to repeat action')\n",
      "cmd:option('-random_starts', 0, 'play action 0 between 1 and random_starts ' ..\n",
      "           'number of times at the start of each training episode')\n",
      "cmd:option('-name', '', 'filename used for saving network and training history')\n",
      "cmd:option('-network', '', 'reload pretrained network')\n",
      "cmd:option('-agent', '', 'name of agent file to use')\n",
      "cmd:option('-agent_params', '', 'string of agent parameters')\n",
      "cmd:option('-seed', 1, 'fixed input seed for repeatable experiments')\n",
      "cmd:option('-saveNetworkParams', false,\n",
      "           'saves the agent network in a separate file')\n",
      "cmd:option('-prog_freq', 5*10^3, 'frequency of progress output')\n",
      "cmd:option('-save_freq', 5*10^4, 'the model is saved every save_freq steps')\n",
      "cmd:option('-eval_freq', 10^4, 'frequency of greedy evaluation')\n",
      "cmd:option('-save_versions', 0, '')\n",
      "\n",
      "cmd:option('-steps', 10^5, 'number of training steps to perform')\n",
      "cmd:option('-eval_steps', 10^5, 'number of evaluation steps')\n",
      "cmd:option('-verbose', 2,\n",
      "           'the higher the level, the more information is printed to screen')\n",
      "cmd:option('-threads', 1, 'number of BLAS threads')\n",
      "cmd:option('-gpu', -1, 'gpu flag')\n",
      "\n",
      "cmd:text()\n",
      "\n",
      "-- Here we parse the table arg to load all command line arguments into opt\n",
      "opt = cmd:parse(arg)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "--\n",
      "-- Observe the parameters that are required to specify the training.\n",
      "--\n",
      "print(opt)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "{\n",
        "  eval_steps : 125000\n",
        "  seed : 1\n",
        "  name : DQN3_0_1_pong_FULL_Y\n",
        "  verbose : 2\n",
        "  network : \n",
        "  pool_frms : type=\"max\",size=2\n",
        "  saveNetworkParams : false\n",
        "  gpu : 0\n",
        "  eval_freq : 250000\n",
        "  env_params : useRGB=true\n",
        "  game_path : /home/ubuntu/DQN/Human_Level_Control_through_Deep_Reinforcement_Learning/roms/\n",
        "  prog_freq : 10000\n",
        "  agent_params : lr=0.00025,ep=1,ep_end=0.1,ep_endt=replay_memory,discount=0.99,hist_len=4,learn_start=50000,replay_memory=1000000,update_freq=4,n_replay=1,network=\"convnet_atari3\",preproc=\"net_downsample_2x_full_y\",state_dim=7056,minibatch_size=32,rescale_r=1,ncols=1,bufferSize=512,valid_size=500,target_q=10000,clip_delta=1,min_reward=-1,max_reward=1\n",
        "  env : pong\n",
        "  framework : alewrap\n",
        "  agent : NeuralQLearner\n",
        "  threads : 4\n",
        "  actrep : 4\n",
        "  random_starts : 30\n",
        "  save_versions : 0\n",
        "  save_freq : 125000\n",
        "  steps : 50000000\n",
        "}\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "-- We do not have too much ram, so make sure to deallocate the current agent.\n",
      "if agent then\n",
      "    agent = nil\n",
      "    collectgarbage()\n",
      "    collectgarbage()\n",
      "end\n",
      "--\n",
      "-- Main setup function\n",
      "-- game : the game environment\n",
      "-- game_actions : valid actions that can be used in this game\n",
      "-- agent : the RL agent that we will train\n",
      "--\n",
      "-- if you don't like too much debug info in ipython\n",
      "--opt.verbose = 0\n",
      "\n",
      "-- run setup to load agent and game\n",
      "game_env, game_actions, agent, opt = setup(opt);"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "Torch Threads:\t4\t\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "Using GPU device id:\t0\t\n",
        "Torch Seed:\t1\t\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "CUTorch Seed:\t1791095845\t\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "\n",
        "Playing:\tpong\t\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "Creating Agent Network from convnet_atari3\t\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "nn.Sequential {\n",
        "  [input -> (1) -> (2) -> (3) -> (4) -> (5) -> (6) -> (7) -> (8) -> (9) -> (10) -> (11) -> output]\n",
        "  (1): nn.Reshape(4x84x84)\n",
        "  (2): nn.SpatialConvolution(in: 4, out: 32, kW: 8, kH: 8, dW: 4, dH: 4, padding: 1)\n",
        "  (3): nn.Rectifier\n",
        "  (4): nn.SpatialConvolution(in: 32, out: 64, kW: 4, kH: 4, dW: 2, dH: 2)\n",
        "  (5): nn.Rectifier\n",
        "  (6): nn.SpatialConvolution(in: 64, out: 64, kW: 3, kH: 3)\n",
        "  (7): nn.Rectifier\n",
        "  (8): nn.Reshape(3136)\n",
        "  (9): nn.Linear(3136 -> 512)\n",
        "  (10): nn.Rectifier\n",
        "  (11): nn.Linear(512 -> 3)\n",
        "}\n",
        "{\n",
        "  gradInput : CudaTensor - empty\n",
        "  modules : \n",
        "    {\n",
        "      1 : \n",
        "        nn.Reshape(4x84x84)\n",
        "        {\n",
        "          nelement : 28224\n",
        "          _input : CudaTensor - empty\n",
        "          output : CudaTensor - size: 1x4x84x84\n",
        "          gradInput : CudaTensor - empty\n",
        "          size : LongStorage - size: 3\n",
        "          _gradOutput : CudaTensor - empty\n",
        "          batchsize : LongStorage - size: 4\n",
        "        }\n",
        "      2 : \n",
        "    "
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "    nn.SpatialConvolution(in: 4, out: 32, kW: 8, kH: 8, dW: 4, dH: 4, padding: 1)\n",
        "        {\n",
        "          dH : 4\n",
        "          dW : 4\n",
        "          nOutputPlane : 32\n",
        "          output : CudaTensor - size: 1x32x20x20\n",
        "          gradInput : CudaTensor - empty\n",
        "          finput : CudaTensor - empty\n",
        "          fgradInput : CudaTensor - empty\n",
        "          gradBias : CudaTensor - size: 32\n",
        "          weight : CudaTensor - size: 32x4x8x8\n",
        "          bias : CudaTensor - size: 32\n",
        "          gradWeight : CudaTensor - size: 32x4x8x8\n",
        "          padding : 1\n",
        "          nInputPlane : 4\n",
        "          kW : 8\n",
        "          kH : 8\n",
        "        }\n",
        "      3 : \n",
        "        nn.Rectifier\n",
        "        {\n",
        "          gradInput : CudaTensor - empty\n",
        "          output : CudaTensor - size: 1x32x20x20\n",
        "        }\n",
        "      4 : \n",
        "        nn.SpatialConvolution(in: 32, out: 64, kW: 4, kH: 4, dW: 2, dH: 2)\n",
        "        {\n",
        "          dH : 2\n",
        "          dW : 2\n",
        "          nOutputPlane : 64\n",
        "          output : CudaTensor - size: 1x64x9x9\n",
        "          gradInput : CudaTensor - empty\n",
        "          finput : CudaTensor - empty\n",
        "          fgradInput : CudaTensor - empty\n",
        "          gradBias : CudaTensor - size: 64\n",
        "          weight : CudaTensor - size: 64x32x4x4\n",
        "          bias : CudaTensor - size: 64\n",
        "          gradWeight : CudaTensor - size: 64x32x4x4\n",
        "          padding : 0\n",
        "          nInputPlane : 32\n",
        "          kW : 4\n",
        "          kH : 4\n",
        "        }\n",
        "      5 : \n",
        "        nn.Rectifier\n",
        "        {\n",
        "          gradInput : CudaTensor - empty\n",
        "          output : CudaTensor - size: 1x64x9x9\n",
        "        }\n",
        "      6 : \n",
        "        nn.SpatialConvolution(in: 64, out: 64, kW: 3, kH: 3)\n",
        "        {\n",
        "          dH : 1\n",
        "          dW : 1\n",
        "          nOutputPlane : 64\n",
        "          output : CudaTensor - size: 1x64x7x7\n",
        "          gradInput : CudaTensor - empty\n",
        " "
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "         finput : CudaTensor - empty\n",
        "          fgradInput : CudaTensor - empty\n",
        "          gradBias : CudaTensor - size: 64\n",
        "          weight : CudaTensor - size: 64x64x3x3\n",
        "          bias : CudaTensor - size: 64\n",
        "          gradWeight : CudaTensor - size: 64x64x3x3\n",
        "          padding : 0\n",
        "          nInputPlane : 64\n",
        "          kW : 3\n",
        "          kH : 3\n",
        "        }\n",
        "      7 : \n",
        "        nn.Rectifier\n",
        "        {\n",
        "          gradInput : CudaTensor - empty\n",
        "          output : CudaTensor - size: 1x64x7x7\n",
        "        }\n",
        "      8 : \n",
        "        nn.Reshape(3136)\n",
        "        {\n",
        "          nelement : 3136\n",
        "          _input : CudaTensor - empty\n",
        "          output : CudaTensor - empty\n",
        "          gradInput : CudaTensor - empty\n",
        "          size : LongStorage - size: 1\n",
        "          _gradOutput : CudaTensor - empty\n",
        "          batchsize : LongStorage - size: 2\n",
        "        }\n",
        "      9 : \n",
        "        nn.Linear(3136 -> 512)\n",
        "        {\n",
        "          gradBias : CudaTensor - size: 512\n",
        "          weight : CudaTensor - size: 512x3136\n",
        "          bias : CudaTensor - size: 512\n",
        "          gradInput : CudaTensor - empty\n",
        "          gradWeight : CudaTensor - size: 512x3136\n",
        "          output : CudaTensor - empty\n",
        "        }\n",
        "      10 : \n",
        "        nn.Rectifier\n",
        "        {\n",
        "          gradInput : CudaTensor - empty\n",
        "          output : CudaTensor - empty\n",
        "        }\n",
        "      11 : \n",
        "        nn.Linear(512 -> 3)\n",
        "        {\n",
        "          gradBias : CudaTensor - size: 3\n",
        "          weight : CudaTensor - size: 3x512\n",
        "          bias : CudaTensor - size: 3\n",
        "          gradInput : CudaTensor - empty\n",
        "          gradWeight : CudaTensor - size: 3x512\n",
        "          output : CudaTensor - empty\n",
        "        }\n",
        "    }\n",
        "  output : CudaTensor - empty\n",
        "}\n",
        "Convolutional layers flattened output size:\t3136\t\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "Set up Torch using these options:\t\n",
        "eval_steps\t125000\t\n",
        "seed\t1\t\n",
        "name\tDQN3_0_1_pong_FULL_Y\t\n",
        "verbose\t2\t\n",
        "network\t\t\n",
        "pool_frms\t{\n",
        "  type : max\n",
        "  size : 2\n",
        "}\n",
        "saveNetworkParams\tfalse\t\n",
        "gpu\t1\t\n",
        "eval_freq\t250000\t\n",
        "tensorType\ttorch.FloatTensor\t\n",
        "env_params\t{\n",
        "  useRGB : true\n",
        "}\n",
        "steps\t50000000\t\n",
        "prog_freq\t10000\t\n",
        "agent_params\t{\n",
        "  target_q : 10000\n",
        "  ncols : 1\n",
        "  replay_memory : 1000000\n",
        "  min_reward : -1\n",
        "  discount : 0.99\n",
        "  bufferSize : 512\n",
        "  hist_len : 4\n",
        "  ep : 1\n",
        "  network : convnet_atari3\n",
        "  max_reward : 1\n",
        "  gpu : 0\n",
        "  n_replay : 1\n",
        "  verbose : 2\n",
        "  ep_end : 0.1\n",
        "  lr : 0.00025\n",
        "  preproc : net_downsample_2x_full_y\n",
        "  valid_size : 500\n",
        "  update_freq : 4\n",
        "  minibatch_size : 32\n",
        "  rescale_r : 1\n",
        "  clip_delta : 1\n",
        "  state_dim : 7056\n",
        "  learn_start : 50000\n",
        "}\n",
        "save_versions\t0\t\n",
        "framework\talewrap\t\n",
        "agent\tNeuralQLearner\t\n",
        "threads\t4\t\n",
        "actrep\t4\t\n",
        "random_starts\t30\t\n",
        "game_path\t/home/ubuntu/DQN/Human_Level_Control_through_Deep_Reinforcement_Learning/roms/\t\n",
        "save_freq\t125000\t\n",
        "env\tpong\t\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print(opt)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "{\n",
        "  eval_steps : 125000\n",
        "  seed : 1\n",
        "  name : DQN3_0_1_pong_FULL_Y\n",
        "  verbose : 2\n",
        "  network : \n",
        "  pool_frms : \n",
        "    {\n",
        "      type : max\n",
        "      size : 2\n",
        "    }\n",
        "  saveNetworkParams : false\n",
        "  gpu : 1\n",
        "  eval_freq : 250000\n",
        "  tensorType : torch.FloatTensor\n",
        "  env_params : \n",
        "    {\n",
        "      useRGB : true\n",
        "    }\n",
        "  steps : 50000000\n",
        "  prog_freq : 10000\n",
        "  agent_params : \n",
        "    {\n",
        "      target_q : 10000\n",
        "      ncols : 1\n",
        "      replay_memory : 1000000\n",
        "      min_reward : -1\n",
        "      discount : 0.99\n",
        "      bufferSize : 512\n",
        "      hist_len : 4\n",
        "      ep : 1\n",
        "      network : convnet_atari3\n",
        "      max_reward : 1\n",
        "      gpu : 0\n",
        "      n_replay : 1\n",
        "      verbose : 2\n",
        "      ep_end : 0.1\n",
        "      lr : 0.00025\n",
        "      preproc : net_downsample_2x_full_y\n",
        "      valid_size : 500\n",
        "      update_freq : 4\n",
        "      minibatch_size : 32\n",
        "      rescale_r : 1\n",
        "      clip_delta : 1\n",
        "      state_dim : 7056\n",
        "      learn_start : 50000\n",
        "    }\n",
        "  save_versions : 0\n",
        "  framework : alewrap\n",
        "  agent : NeuralQLearner\n",
        "  threads : 4\n",
        "  actrep : 4\n",
        "  random_starts : 30\n",
        "  game_path : /home/ubuntu/DQN/Human_Level_Control_through_Deep_Reinforcement_Learning/roms/\n",
        "  save_freq : 125000\n",
        "  env : pong\n",
        "}\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print(game_actions)\n",
      "screens = {}\n",
      "for i=1,36 do\n",
      "    screen, reward, terminal = game_env:step(0)\n",
      "    table.insert(screens, screen[1]:clone())\n",
      "end\n",
      "itorch.image(screens)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "{\n",
        "  1 : 0\n",
        "  2 : 3\n",
        "  3 : 4\n",
        "}\n"
       ]
      },
      {
       "metadata": {
        "png": {
         "height": 1272,
         "width": 972
        }
       },
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAA8wAAAT4CAIAAABCBtwMAAAgAElEQVR4nO3cQWpbW6KG0aOHRnA9GXeMB2CuwXPQLNz2KOI5GAQZQEjHk9EY9BpV5OVVJbGv/R0d6ey1Woa40IafS33Z7GhzPB4nAACgs52mabPZ/PLPvvz912kPwyx2+8Nk5bWz8gisPAIrj8DKI9jtD9vptHPe3d68+Ttfv30/wUlO4/Xx4c3fuX56OcFJrDwfK/+Oledg5flY+XesPAcrz+dMVv6fuT8AAABGI7IBACAmsgEAILZd8LN/fv3zntdCl+7n1z/veS20DlYegZVHYOURWHkEVj4ZN9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAENsu+Nl3tzcLfvrpvT4+LH2EBVh5BFYegZVHYOURWPlk3GQDAEBMZAMAQExkAwBAbHM8Hp/vr5Y+BjPa7Q/TNFl53aw8AiuPwMojsPIIdvuDm2wAAIiJbAAAiG2Ox+PSZwAAgFVxkw0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAACx7TRNz/dXSx+DGe32h8nKa2flEVh5BFYegZVHsNsf3GQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBsu/QBeJfXx4cfP18/vSx4EuZj5RFYeQRWHoGVR/DJld1kAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQGy79AF4l+unl6WPwOysPAIrj8DKI7DyCD65sptsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiG2Ox+PSZwAAgFVxkw0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAse00TZvN5pd/9uXvv057GGax2x8mK6+dlUdg5RFYeQRWHsFuf9hOp53z7vbmzd/5+u37CU5yGq+PD2/+zvXTywlOYuX5WPl3rDwHK8/Hyr9j5TlYeT5nsrLnIgAAEBPZAAAQE9kAABDbLvjZP7/+ec9roUv38+uf97wWWgcrj8DKI7DyCKw8AiufjJtsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIhtF/zsu9ubBT/99F4fH5Y+wgKsPAIrj8DKI7DyCKx8Mm6yAQAgJrIBACAmsgEAILY5Ho/P91dLH4MZ7faHaZqsvG5WHoGVR2DlEVh5BLv9wU02AADERDYAAMQ2x+Nx6TMAAMCquMkGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCA2Haapuf7q6WPwYx2+8Nk5bWz8gisPAIrj8DKI9jtD26yAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgtl36ALzL6+PDj5+vn14WPAnzsfIIrDwCK4/AyiP45MpusgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACC2XfoAvMv108vSR2B2Vh6BlUdg5RFYeQSfXNlNNgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMQ2x+Nx6TMAAMCquMkGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgNh2mqbNZvPLP/vy91+nPQyz2O0Pk5XXzsojsPIIrDwCK49gtz9sp9POeXd78+bvfP32/QQnOY3Xx4c3f+f66eUEJ7HyfKz8O1aeg5XnY+XfsfIcrDyfM1nZcxEAAIiJbAAAiIlsAACIbRf87J9f/7zntdCl+/n1z3teC62DlUdg5RFYeQRWHoGVT8ZNNgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADEtgt+9t3tzYKffnqvjw9LH2EBVh6BlUdg5RFYeQRWPhk32QAAEBPZAAAQE9kAABDbHI/H5/urpY/BjHb7wzRNVl43K4/AyiOw8gisPILd/uAmGwAAYiIbAABim+PxuPQZAABgVdxkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQGw7TdPz/dXSx2BGu/1hsvLaWXkEVh6BlUdg5RHs9gc32QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAENsufQDe5fXx4cfP108vC56E+Vh5BFYegZVHYOURfHJlN9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQ2y59AN7l+ull6SMwOyuPwMojsPIIrDyCT67sJhsAAGIiGwAAYp6LwLjubm9+/Pz12/cFTwIAK+MmG7gwu/1htz8sfQoA+BORDVwSeQ3ARRDZAAAQ8yYbYM28vAdYhJts4PI8318tfQR6XtsDa+ImG7gk8nqt5DWwMm6yAQAg5iYbAC6bl/dwhtxkA3AuPAdaJa/tGZObbACWJ6/XSl4zLDfZAAAQc5MNAHDuvLy/OG6yAYB5eQ60Sl7b/5mbbABgLvJ6reT1m9xkAwBATGQDAEBMZAMA8BGeA/2BN9kwLv8+HYCPkddvcpMNAAAxkQ0AADGRDQAAMW+yAdbMy3uARbjJBgCAmMgGAICYyAYAgJg32QBw2by8hzPkJhsAAGIiGwAAYiIbAABi3mQDAJw7L+8vjptsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiG2Ox+PSZwAAgFVxkw0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAse00TZvN5pd/9uXvv057GGax2x8mK6+dlUdg5RFYeQRWHsFuf9hOp53z7vbmzd/5+u37CU5yGq+PD2/+zvXTywlOYuX5WPl3rDwHK8/Hyr9j5TlYeT5nsrLnIgAAEBPZAAAQE9kAABDbLvjZP7/+ec9roUv38+uf97wWWgcrj8DKI7DyCKw8AiufjJtsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIhtF/zsu9ubBT/99F4fH5Y+wgKsPAIrj8DKI7DyCKx8Mm6yAQAgJrIBACAmsgEAILY5Ho/P91dLH4MZ7faHaZqsvG5WHoGVR2DlEVh5BLv9wU02AADERDYAAMQ2x+Nx6TMAAMCquMkGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCA2Haapuf7q6WPwYx2+8Nk5bWz8gisPAIrj8DKI9jtD26yAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgtl36ALzL6+PDj5+vn14WPAnzsfIIrDwCK4/AyiP45MpusgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACC2XfoAvMv108vSR2B2Vh6BlUdg5RFYeQSfXNlNNgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMS2H/jf3N3e/Pj567fv3WEAAGANLvgme7c/7PaHpU8BAAD/6VIjW14DAHC2LjWyAQDgbH3kTTYj8PIeAODDLvsm+/n+aukj0PPaHgC4dJd6ky2v10peAwArcNk32QAAcIYu9SYb+Dwv7wFgJm6yOUeeA62S1/YAjMNNNudFXq+VvAZgKG6yAQAg5iYbYM28vAdYhJts4HQ8B1olr+0B/pubbOAU5PVayWuAX3KTDQAAMTfZAHDZvLyHM+QmG4DP8hxolby2h8/4yE22vyUD8C/yeq3kNXySm2wAAIh5kw0AcO68vL84brIBgF/zHGiVvLY/DTfZ/Jq/JQOMTF6vlbw+GTfZAAAQE9kAABAT2QAAY/Ec6AS8yYZxeXkPMBp5fTJusgEAICayAQAgJrIBACDmTTbAmnl5D7AIN9kAABAT2QAAEBPZAAAQ8yYbAC6bl/dwhtxkAwBATGQDAEBMZAMAQMybbACAc+fl/cVxkw0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAACxzfF4XPoMAACwKm6yAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACC2naZps9n88s++/P3XaQ/DLHb7w2TltbPyCKw8AiuPwMoj2O0P2+m0c97d3rz5O1+/fT/BSU7j9fHhzd+5fno5wUmsPB8r/46V52Dl+Vj5d6w8ByvP50xW9lwEAABiIhsAAGIiGwAAYtsFP/vn1z/veS106X5+/fOe10LrYOURWHkEVh6BlUdg5ZNxkw0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAse2Cn313e7Pgp5/e6+PD0kdYgJVHYOURWHkEVh6BlU/GTTYAAMRENgAAxEQ2AADENsfj8fn+auljMKPd/jBNk5XXzcojsPIIrDwCK49gtz+4yQYAgJjIBgCA2OZ4PC59BgAAWBU32QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABDbTtP0fH+19DGY0W5/mKy8dlYegZVHYOURWHkEu/3BTTYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMS2Sx+Ad3l9fPjx8/XTy4InYT5WHoGVR2DlEVh5BJ9c2U02AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxLZLH4B3uX56WfoIzM7KI7DyCKw8AiuP4JMru8kGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCA2PYD/5u725sfP3/99r07DAAArMECN9m7/WG3P5z+cwEA4DROHdnyGgCA1fMmGwAAYh95k80IvLwHAPiwZW6yn++vFvlcZuW1PQDAv5z6Jlter5W8BgD4wZtsAACIeZMN4/LyHgBm4iabkudAq+S1PQD8U26yacjrtZLXAPABbrIBACDmJhtgzby8h3Xw3/LFcZMNvM1zoFXy2h4umv+Ez5ybbOBP5PVa+f9mgFm5yQYAuDD+nnz+3GQDwGXzWhfOkJtsgHF5DrRKnuqOw3/C5+wjN9n+lgxw6fx/81rJazgTnosAAFwYf08+f56LAABATGQDwNq45oTFeS7Cr3l5D3CJ5DWcCTfZAAAQE9kAABAT2QAAEPMmG8bl5T0AzMRNNgAAxEQ2AADERDYAAMS8yQZYMy/vYR38t3xx3GQDAEBMZAMAQExkAwBAzJtsALhsXuvCGXKTDQAAMZENAAAxkQ0AADGRDQAAsc3xeFz6DAAAsCpusgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgtp2mabPZ/PLPvvz912kPwyx2+8Nk5bWz8gisPAIrj8DKI9jtD9vptHPe3d68+Ttfv30/wUlO4/Xx4c3fuX56OcFJrDwfK/+Oledg5flY+XesPAcrz+dMVvZcBAAAYiIbAABiIhsAAGLbBT/759c/73ktdOl+fv3zntdC62DlEVh5BFYegZVHYOWTcZMNAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AALHtgp99d3uz4Kef3uvjw9JHWICVR2DlEVh5BFYegZVPxk02AADERDYAAMRENgAAxDbH4/H5/mrpYzCj3f4wTZOV183KI7DyCKw8AiuPYLc/uMkGAICYyAYAgNjmeDwufQYAAFgVN9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQ207T9Hx/tfQxmNFuf5isvHZWHoGVR2DlEVh5BLv9wU02AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADEtksfgHd5fXz48fP108uCJ2E+Vh6BlUdg5RFYeQSfXNlNNgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMS2Sx+Ad7l+eln6CMzOyiOw8gisPAIrj+CTK7vJBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGABjXbn/Y7Q9Ln2KFRDYAAMRENgDAoNxhz0dkAwBATGQDAAzt+f5q6SOs0HbpAwAA8Ia725sfP3/99n3Bk/BOIhsAYFDusOfjuQgAAMRENgDwf3xrMiRENgAAxEQ2APBv7rChIrIBACAmsgGA/8c3TsDn+Qo/ALhsvkEZzpDIBgD+zR02VDwXAQCAmMgG4E98azLAB4hsAACIiWwAfssdNsDH+IePAGvmeycAFuEmG4A3+MYJgH9KZAMAQMxzEQB+yx02wMeIbACAc+ffVFwcz0WAf8a3JgPAm0Q2AADERDbwD7jDBoD38CYbxuUblAFgJm6ygX/MN04AwJ+JbAAAiHkuAvwD7rAB4D1ENgBcNv+mAs6Q55EQplcAAAFxSURBVCJ8lm9NBgD4D26y+TXfOwEA8GFusvkUd9gAAP9NZAMAQExkE/CNEwAAP/MmG2DN/JsKgEWIbD7FHTYAwH/zXAQAAGJ9ZPvWZAAABveR5yK+QRkAAP4gvsl2hw0AAN5kAwBAbJbI9o0TAACMzFf4wbj8mwoAmEkc2e6wAQDAm2wAAIiJbAAAiIlsAACIiWwAAIj5dhF+zfdOAAB8mJtsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIfeR7sn2DMgAA/IGbbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIbY7H49JnAACAVXGTDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAsf8FhEByBIOznGUAAAAASUVORK5CYII=",
       "text": [
        "Console does not support images"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "agent"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "dqn.NeuralQLearner\n",
        "{\n",
        "  lr_endt : 1000000\n",
        "  replay_memory : 1000000\n",
        "  bufferSize : 512\n",
        "  hist_len : 4\n",
        "  lr_start : 0.00025\n",
        "  histType : linear\n",
        "  bestq : 0\n",
        "  histSpacing : 1\n",
        "  deltas : CudaTensor - size: 1685667\n",
        "  input_dims : \n",
        "    {\n",
        "      1 : 4\n",
        "      2 : 84\n",
        "      3 : 84\n",
        "    }\n",
        "  n_hid : \n",
        "    {\n",
        "      1 : 512\n",
        "    }\n",
        "  ep_start : 1\n",
        "  q_max : 1\n",
        "  g : CudaTensor - size: 1685667\n",
        "  target_network : \n",
        "    nn.Sequential {\n",
        "      [input -> (1) -> (2) -> (3) -> (4) -> (5) -> (6) -> (7) -> (8) -> (9) -> (10) -> (11) -> output]\n",
        "      (1): nn.Reshape(4x84x84)\n",
        "      (2): nn.SpatialConvolution(in: 4, out: 32, kW: 8, kH: 8, dW: 4, dH: 4, padding: 1)\n",
        "      (3): nn.Rectifier\n",
        "      (4): nn.SpatialConvolution(in: 32, out: 64, kW: 4, kH: 4, dW: 2, dH: 2)\n",
        "      (5): nn.Rectifier\n",
        "      (6): nn.SpatialConvolution(in: 64, out: 64, kW: 3, kH: 3)\n",
        "      (7): nn.Rectifier\n",
        "      (8): nn.Reshape(3136)\n",
        "      (9): nn.Linear(3136 -> 512)\n",
        "      (10): nn.Rectifier\n",
        "      (11): nn.Linear(512 -> 3)\n",
        "    }\n",
        "    {\n",
        "      gradInput : CudaTensor - empty\n",
        "      modules : \n",
        "        {\n",
        "          1 : \n",
        "            nn.Reshape(4x84x84)\n",
        "            {\n",
        "              batchsize : LongStorage - size: 4\n",
        "              size : LongStorage - size: 3\n",
        "              output : CudaTensor - size: 1x4x84x84\n",
        "              gradInput : CudaTensor - empty\n",
        "              nelement : 28224\n",
        "              _gradOutput : CudaTensor - empty\n",
        "              _input : CudaTensor - empty\n",
        "            }\n",
        "          2 : \n",
        "            nn.SpatialConvolution(in: 4, out: 32, kW: 8, kH: 8, dW: 4, dH: 4, padding: 1)\n",
        "            {\n",
        "              dH : 4\n",
        "              dW : 4\n",
        "         "
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "     nOutputPlane : 32\n",
        "              output : CudaTensor - size: 1x32x20x20\n",
        "              gradInput : CudaTensor - empty\n",
        "              finput : CudaTensor - empty\n",
        "              kW : 8\n",
        "              kH : 8\n",
        "              weight : CudaTensor - size: 32x4x8x8\n",
        "              nInputPlane : 4\n",
        "              gradWeight : CudaTensor - size: 32x4x8x8\n",
        "              padding : 1\n",
        "              bias : CudaTensor - size: 32\n",
        "              gradBias : CudaTensor - size: 32\n",
        "              fgradInput : CudaTensor - empty\n",
        "            }\n",
        "          3 : \n",
        "            nn.Rectifier\n",
        "            {\n",
        "              gradInput : CudaTensor - empty\n",
        "              output : CudaTensor - size: 1x32x20x20\n",
        "            }\n",
        "          4 : \n",
        "            nn.SpatialConvolution(in: 32, out: 64, kW: 4, kH: 4, dW: 2, dH: 2)\n",
        "            {\n",
        "              dH : 2\n",
        "              dW : 2\n",
        "              nOutputPlane : 64\n",
        "              output : CudaTensor - size: 1x64x9x9\n",
        "              gradInput : CudaTensor - empty\n",
        "              finput : CudaTensor - empty\n",
        "              kW : 4\n",
        "              kH : 4\n",
        "              weight : CudaTensor - size: 64x32x4x4\n",
        "              nInputPlane : 32\n",
        "              gradWeight : CudaTensor - size: 64x32x4x4\n",
        "              padding : 0\n",
        "              bias : CudaTensor - size: 64\n",
        "              gradBias : CudaTensor - size: 64\n",
        "              fgradInput : CudaTensor - empty\n",
        "       "
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "     }\n",
        "          5 : \n",
        "            nn.Rectifier\n",
        "            {\n",
        "              gradInput : CudaTensor - empty\n",
        "              output : CudaTensor - size: 1x64x9x9\n",
        "            }\n",
        "          6 : \n",
        "            nn.SpatialConvolution(in: 64, out: 64, kW: 3, kH: 3)\n",
        "            {\n",
        "              dH : 1\n",
        "              dW : 1\n",
        "              nOutputPlane : 64\n",
        "              output : CudaTensor - size: 1x64x7x7\n",
        "              gradInput : CudaTensor - empty\n",
        "              finput : CudaTensor - empty\n",
        "              kW : 3\n",
        "              kH : 3\n",
        "              weight : CudaTensor - size: 64x64x3x3\n",
        "              nInputPlane : 64\n",
        "              gradWeight : CudaTensor - size: 64x64x3x3\n",
        "              padding : 0\n",
        "              bias : CudaTensor - size: 64\n",
        "              gradBias : CudaTensor - size: 64\n",
        "              fgradInput : CudaTensor - empty\n",
        "            }\n",
        "          7 : \n",
        "            nn.Rectifier\n",
        "            {\n",
        "              gradInput : CudaTensor - empty\n",
        "              output : CudaTensor - size: 1x64x7x7\n",
        "            }\n",
        "          8 : \n",
        "            nn.Reshape(3136)\n",
        "            {\n",
        "              batchsize : LongStorage - size: 2\n",
        "              size : LongStorage - size: 1\n",
        "              output : CudaTensor - empty\n",
        "              gradInput : CudaTensor - empty\n",
        "              nelement : 3136\n",
        "              _gradOutput : CudaTensor - empty\n",
        "              _input : CudaTensor - empty\n",
        "            }\n",
        "          9 : \n",
        "            nn.Linear(3136 -> 512)\n",
        "            {\n",
        "              gradBias : CudaTensor - size: 512\n",
        "              weight : CudaTensor - size: 512x3136\n",
        "              bias : CudaTensor - size: 512\n",
        "              gradInput : CudaTensor - empty\n",
        "              output : CudaTensor - empty\n",
        "              gradWeight : CudaTensor - size: 512x3136\n",
        "            }\n",
        "        "
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "  10 : \n",
        "            nn.Rectifier\n",
        "            {\n",
        "              gradInput : CudaTensor - empty\n",
        "              output : CudaTensor - empty\n",
        "            }\n",
        "          11 : \n",
        "            nn.Linear(512 -> 3)\n",
        "            {\n",
        "              gradBias : CudaTensor - size: 3\n",
        "              weight : CudaTensor - size: 3x512\n",
        "              bias : CudaTensor - size: 3\n",
        "              gradInput : CudaTensor - empty\n",
        "              output : CudaTensor - empty\n",
        "              gradWeight : CudaTensor - size: 3x512\n",
        "            }\n",
        "        }\n",
        "      output : CudaTensor - empty\n",
        "    }\n",
        "  ep_end : 0.1\n",
        "  g2 : CudaTensor - size: 1685667\n",
        "  w : CudaTensor - size: 1685667\n",
        "  update_freq : 4\n",
        "  minibatch_size : 32\n",
        "  ncols : 1\n",
        "  r_max : 1\n",
        "  transition_params : table: 0x41621680\n",
        "  lr_end : 0.00025\n",
        "  target_q : 10000\n",
        "  numSteps : 0\n",
        "  v_avg : 0\n",
        "  n_units : \n",
        "    {\n",
        "      1 : 32\n",
        "      2 : 64\n",
        "      3 : 64\n",
        "    }\n",
        "  ep_endt : 1000000\n",
        "  min_reward : -1\n",
        "  discount : 0.99\n",
        "  learn_start : 50000\n",
        "  filter_stride : \n",
        "    {\n",
        "      1 : 4\n",
        "      2 : 2\n",
        "      3 : 1\n",
        "    }\n",
        "  ep : 1\n",
        "  tderr_avg : 0\n",
        "  transitions : \n",
        "    dqn.TransitionTable\n",
        "    {\n",
        "      stateDim : 7056\n",
        "      histLen : 4\n",
        "      recentMemSize : 4\n",
        "      nonEventProb : 1\n",
        "      maxSize : 1000000\n",
        "      bufferSize : 512\n",
        "      a : LongTensor - size: 1000000\n",
        "      gpu_s2 : CudaTensor - size: 512x28224\n",
        "      histType : linear\n",
        "      gpu_s : CudaTensor - size: 512x28224\n",
        "      insertIndex : 0\n",
        "      histSpacing : 1\n",
        "      buf_s : ByteTensor - size: 512x28224\n",
        "      buf_s2 : ByteTensor - size: 512x28224\n",
        "      gpu : 0\n",
        "      numActions : 3\n",
        "      nonTermProb : 1\n",
        "      s : ByteTensor - size: 1000000x7056\n",
        "      action_encodings : FloatTensor - size: 3x3\n",
        "      buf_term : ByteTensor - size: 512\n",
        "      buf_r : FloatTensor - size: 512\n",
        "      t : ByteTensor - size: 1000000\n",
        "      buf_a : LongTensor - size: 512\n",
        "      recent_t : table: 0x41d7e5b8\n",
        "      recent_a : table: 0x41d7e590\n",
        "      recent_s : table: 0x41d7e568\n",
        "      r : FloatTensor - size: 1000000\n",
        "      zeroFrames : 1\n",
        "      numEntries : 0\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "      histIndices : \n",
        "        {\n",
        "          1 : 1\n",
        "          2 : 2\n",
        "          3 : 3\n",
        "          4 : 4\n",
        "        }\n",
        "    }\n",
        "  max_reward : 1\n",
        "  wc : 0\n",
        "  tensor_type : table: 0x40d6f718\n",
        "  nl : table: 0x41d65620\n",
        "  actions : \n",
        "    {\n",
        "      1 : 0\n",
        "      2 : 3\n",
        "      3 : 4\n",
        "    }\n",
        "  n_replay : 1\n",
        "  nonTermProb : 1\n",
        "  tmp : CudaTensor - size: 1685667\n",
        "  network : \n",
        "    nn.Sequential {\n",
        "      [input -> (1) -> (2) -> (3) -> (4) -> (5) -> (6) -> (7) -> (8) -> (9) -> (10) -> (11) -> output]\n",
        "      (1): nn.Reshape(4x84x84)\n",
        "      (2): nn.SpatialConvolution(in: 4, out: 32, kW: 8, kH: 8, dW: 4, dH: 4, padding: 1)\n",
        "      (3): nn.Rectifier\n",
        "      (4): nn.SpatialConvolution(in: 32, out: 64, kW: 4, kH: 4, dW: 2, dH: 2)\n",
        "      (5): nn.Rectifier\n",
        "      (6): nn.SpatialConvolution(in: 64, out: 64, kW: 3, kH: 3)\n",
        "      (7): nn.Rectifier\n",
        "      (8): nn.Reshape(3136)\n",
        "      (9): nn.Linear(3136 -> 512)\n",
        "      (10): nn.Rectifier\n",
        "      (11): nn.Linear(512 -> 3)\n",
        "    }\n",
        "    {\n",
        "      gradInput : CudaTensor - empty\n",
        "      modules : \n",
        "        {\n",
        "          1 : \n",
        "            nn.Reshape(4x84x84)\n",
        "            {\n",
        "              nelement : 28224\n",
        "              _input : CudaTensor - empty\n",
        "              output : CudaTensor - size: 1x4x84x84\n",
        "              gradInput : CudaTensor - empty\n",
        "              size : LongStorage - size: 3\n",
        "              _gradOutput : CudaTensor - empty\n",
        "              batchsize : LongStorage - size: 4\n",
        "            }\n",
        "          2 : \n",
        "            nn.SpatialConvolution(in: 4, out: 32, kW: 8, kH: 8, dW: 4, dH: 4, padding: 1)\n",
        "            {\n",
        "              dH : 4\n",
        "              dW : 4\n",
        "              nOutputPlane : 32\n",
        "              output : CudaTensor - size: 1x32x20x20\n",
        "              gradInput : CudaTensor - empty\n",
        "              finput : CudaTensor - empty\n",
        "              fgradInput : CudaTensor - empty\n",
        "              gradBias : CudaTensor - size: 32\n",
        "              weight : CudaTensor - size: 32x4x8x8\n",
        "              bias : CudaTensor - size: 32\n",
        "              gradWeight : CudaTensor - size: 32x4x8x8\n",
        "              padding : 1\n",
        "              nInputPlane : 4\n",
        "              kW : 8\n",
        "              kH : 8\n",
        "            }\n",
        " "
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "         3 : \n",
        "            nn.Rectifier\n",
        "            {\n",
        "              gradInput : CudaTensor - empty\n",
        "              output : CudaTensor - size: 1x32x20x20\n",
        "            }\n",
        "          4 : \n",
        "            nn.SpatialConvolution(in: 32, out: 64, kW: 4, kH: 4, dW: 2, dH: 2)\n",
        "            {\n",
        "              dH : 2\n",
        "              dW : 2\n",
        "              nOutputPlane : 64\n",
        "              output : CudaTensor - size: 1x64x9x9\n",
        "              gradInput : CudaTensor - empty\n",
        "              finput : CudaTensor - empty\n",
        "              fgradInput : CudaTensor - empty\n",
        "              gradBias : CudaTensor - size: 64\n",
        "              weight : CudaTensor - size: 64x32x4x4\n",
        "              bias : CudaTensor - size: 64\n",
        "              gradWeight : CudaTensor - size: 64x32x4x4\n",
        "              padding : 0\n",
        "              nInputPlane : 32\n",
        "              kW : 4\n",
        "              kH : 4\n",
        "            }\n",
        "          5 : \n",
        "            nn.Rectifier\n",
        "            {\n",
        "              gradInput : CudaTensor - empty\n",
        "              output : CudaTensor - size: 1x64x9x9\n",
        "            }\n",
        "          6 : \n",
        "            nn.SpatialConvolution(in: 64, out: 64, kW: 3, kH: 3)\n",
        "            {\n",
        "              dH : 1\n",
        "              dW : 1\n",
        "              nOutputPlane : 64\n",
        "              output : CudaTensor - size: 1x64x7x7\n",
        "              gradInput : CudaTensor - empty\n",
        "              finput : CudaTensor - empty\n",
        "              fgradInput : CudaTensor - empty\n",
        "              gradBias : CudaTensor - size: 64\n",
        "              weight : CudaTensor - size: 64x64x3x3\n",
        "              bias : CudaTensor - size: 64\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "              gradWeight : CudaTensor - size: 64x64x3x3\n",
        "              padding : 0\n",
        "              nInputPlane : 64\n",
        "              kW : 3\n",
        "              kH : 3\n",
        "            }\n",
        "          7 : \n",
        "            nn.Rectifier\n",
        "            {\n",
        "              gradInput : CudaTensor - empty\n",
        "              output : CudaTensor - size: 1x64x7x7\n",
        "            }\n",
        "          8 : \n",
        "            nn.Reshape(3136)\n",
        "            {\n",
        "              nelement : 3136\n",
        "              _input : CudaTensor - empty\n",
        "              output : CudaTensor - empty\n",
        "              gradInput : CudaTensor - empty\n",
        "              size : LongStorage - size: 1\n",
        "              _gradOutput : CudaTensor - empty\n",
        "              batchsize : LongStorage - size: 2\n",
        "            }\n",
        "          9 : \n",
        "            nn.Linear(3136 -> 512)\n",
        "            {\n",
        "              gradBias : CudaTensor - size: 512\n",
        "              weight : CudaTensor - size: 512x3136\n",
        "              bias : CudaTensor - size: 512\n",
        "              gradInput : CudaTensor - empty\n",
        "              gradWeight : CudaTensor - size: 512x3136\n",
        "              output : CudaTensor - empty\n",
        "            }\n",
        "          10 : \n",
        "            nn.Rectifier\n",
        "            {\n",
        "              gradInput : CudaTensor - empty\n",
        "              output : CudaTensor - empty\n",
        "            }\n",
        "          11 : \n",
        "            nn.Linear(512 -> 3)\n",
        "            {\n",
        "              gradBias : CudaTensor - size: 3\n",
        "              weight : CudaTensor - size: 3x512\n",
        "              bias : CudaTensor - size: 3\n",
        "              gradInput : CudaTensor - empty\n",
        "              gradWeight : CudaTensor - size: 3x512\n",
        "              output : CudaTensor - empty\n",
        "            }\n",
        "        }\n",
        "      "
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "output : CudaTensor - empty\n",
        "    }\n",
        "  gpu : 0\n",
        "  filter_size : \n",
        "    {\n",
        "      1 : 8\n",
        "      2 : 4\n",
        "      3 : 3\n",
        "    }\n",
        "  lr : 0.00025\n",
        "  preproc : \n",
        "    nn.Scale\n",
        "    {\n",
        "      height : 84\n",
        "      width : 84\n",
        "    }\n",
        "  valid_size : 500\n",
        "  clip_delta : 1\n",
        "  verbose : 2\n",
        "  rescale_r : 1\n",
        "  n_actions : 3\n",
        "  state_dim : 7056\n",
        "  dw : CudaTensor - size: 1685667\n",
        "}\n"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "local learn_start = agent.learn_start\n",
      "local start_time = sys.clock()\n",
      "local reward_counts = {}\n",
      "local episode_counts = {}\n",
      "local time_history = {}\n",
      "local v_history = {}\n",
      "local qmax_history = {}\n",
      "local td_history = {}\n",
      "local reward_history = {}\n",
      "local step = 0\n",
      "time_history[1] = 0\n",
      "\n",
      "local total_reward\n",
      "local nrewards\n",
      "local nepisodes\n",
      "local episode_reward\n",
      "\n",
      "\n",
      "-- When we save agent, there are many temporary states in the agent. The proper way would be to implement \n",
      "-- agent:read() and agent:write() functions as in TransitionTable.lua \n",
      "-- But using Lua closures, we can also have a nice interface like follows.\n",
      "-- agent is the agent to make slim so that we do not dump GBs of data\n",
      "-- returns a deslim function that does not take any arguments and reinstates the parameters in the agent.\n",
      "local function slim_agent(agent)\n",
      "    local s, a, r, s2, term = agent.valid_s, agent.valid_a, agent.valid_r,agent.valid_s2, agent.valid_term\n",
      "    agent.valid_s, agent.valid_a, agent.valid_r, agent.valid_s2, agent.valid_term = nil, nil, nil, nil, nil, nil, nil\n",
      "    local w, dw, g, g2, delta, delta2, deltas, tmp = agent.w, agent.dw, agent.g, agent.g2, agent.delta, agent.delta2, agent.deltas, agent.tmp\n",
      "    agent.w, agent.dw, agent.g, agent.g2, agent.delta, agent.delta2, agent.deltas, agent.tmp = nil, nil, nil, nil, nil, nil, nil, nil\n",
      "    \n",
      "    -- this function now has pointers to all the local variables here when it was called.\n",
      "    -- so it can remodify the agent with the correct values. One could call slim_agent with many different\n",
      "    -- agent instances, and every deslim function would point to the correct context.\n",
      "    local function deslim()\n",
      "        agent.valid_s, agent.valid_a, agent.valid_r, agent.valid_s2, agent.valid_term = s, a, r, s2, term\n",
      "        agent.w, agent.dw, agent.g, agent.g2, agent.delta, agent.delta2, agent.deltas, agent.tmp = w, dw, g, g2, delta, delta2, deltas, tmp\n",
      "    end\n",
      "    return deslim\n",
      "end\n",
      "\n",
      "local screen, reward, terminal = game_env:getState()\n",
      "\n",
      "print(\"Iteration ..\", step)\n",
      "while step < opt.steps do\n",
      "    step = step + 1\n",
      "    local action_index = agent:perceive(reward, screen, terminal)\n",
      "\n",
      "    -- game over? get next game!                                                                                                                      \n",
      "    if not terminal then\n",
      "        screen, reward, terminal = game_env:step(game_actions[action_index], true)\n",
      "    else\n",
      "        if opt.random_starts > 0 then\n",
      "            screen, reward, terminal = game_env:nextRandomGame()\n",
      "        else\n",
      "            screen, reward, terminal = game_env:newGame()\n",
      "        end\n",
      "    end\n",
      "\n",
      "    if step % opt.prog_freq == 0 then\n",
      "        assert(step==agent.numSteps, 'trainer step: ' .. step ..\n",
      "                ' & agent.numSteps: ' .. agent.numSteps)\n",
      "        print(\"Training Steps: \", step)\n",
      "        --agent:report()\n",
      "        collectgarbage()\n",
      "    end\n",
      "\n",
      "    if step%1000 == 0 then collectgarbage() end\n",
      "\n",
      "    if step % opt.eval_freq == 0 and step > learn_start then\n",
      "\n",
      "        print('Evaluating')\n",
      "        screen, reward, terminal = game_env:newGame()\n",
      "\n",
      "        total_reward = 0\n",
      "        nrewards = 0\n",
      "        nepisodes = 0\n",
      "        episode_reward = 0\n",
      "\n",
      "        local eval_time = sys.clock()\n",
      "        for estep=1,opt.eval_steps do\n",
      "            local action_index = agent:perceive(reward, screen, terminal, true, 0.05)\n",
      "            -- Play game in test mode (episodes don't end when losing a life)                                                                         \n",
      "            screen, reward, terminal = game_env:step(game_actions[action_index])\n",
      "\n",
      "            if estep%1000 == 0 then collectgarbage() end\n",
      "\n",
      "            -- record every reward                                                                                                                    \n",
      "            episode_reward = episode_reward + reward\n",
      "            if reward ~= 0 then\n",
      "               nrewards = nrewards + 1\n",
      "            end\n",
      "\n",
      "            if terminal then\n",
      "                total_reward = total_reward + episode_reward\n",
      "                episode_reward = 0\n",
      "                nepisodes = nepisodes + 1\n",
      "                screen, reward, terminal = game_env:nextRandomGame()\n",
      "            end\n",
      "        end\n",
      "\n",
      "        eval_time = sys.clock() - eval_time\n",
      "        start_time = start_time + eval_time\n",
      "        agent:compute_validation_statistics()\n",
      "        local ind = #reward_history+1\n",
      "        total_reward = total_reward/math.max(1, nepisodes)\n",
      "\n",
      "        if #reward_history == 0 or total_reward > torch.Tensor(reward_history):max() then\n",
      "            agent.best_network = agent.network:clone()\n",
      "        end\n",
      "\n",
      "        if agent.v_avg then\n",
      "            v_history[ind] = agent.v_avg\n",
      "            td_history[ind] = agent.tderr_avg\n",
      "            qmax_history[ind] = agent.q_max\n",
      "        end\n",
      "        print(\"V\", v_history[ind], \"TD error\", td_history[ind], \"Qmax\", qmax_history[ind])\n",
      "\n",
      "        reward_history[ind] = total_reward\n",
      "        reward_counts[ind] = nrewards\n",
      "        episode_counts[ind] = nepisodes\n",
      "        time_history[ind+1] = sys.clock() - start_time\n",
      "\n",
      "        local time_dif = time_history[ind+1] - time_history[ind]\n",
      "\n",
      "        local training_rate = opt.actrep*opt.eval_freq/time_dif\n",
      "\n",
      "        print(string.format(\n",
      "            '\\nSteps: %d (frames: %d), reward: %.2f, epsilon: %.2f, lr: %G, ' ..\n",
      "            'training time: %ds, training rate: %dfps, testing time: %ds, ' ..\n",
      "            'testing rate: %dfps,  num. ep.: %d,  num. rewards: %d',\n",
      "            step, step*opt.actrep, total_reward, agent.ep, agent.lr, time_dif,\n",
      "            training_rate, eval_time, opt.actrep*opt.eval_steps/eval_time,\n",
      "            nepisodes, nrewards))\n",
      "    end\n",
      "\n",
      "    if step % opt.save_freq == 0 or step == opt.steps then\n",
      "        \n",
      "        local deslim_agent = slim_agent(agent)\n",
      "\n",
      "        local filename = opt.name\n",
      "        if opt.save_versions > 0 then\n",
      "            filename = filename .. \"_\" .. math.floor(step / opt.save_versions)\n",
      "        end\n",
      "        filename = filename\n",
      "        torch.save(filename .. \".t7\", {agent = agent,\n",
      "                                model = agent.network,\n",
      "                                best_model = agent.best_network,\n",
      "                                reward_history = reward_history,\n",
      "                                reward_counts = reward_counts,\n",
      "                                episode_counts = episode_counts,\n",
      "                                time_history = time_history,\n",
      "                                v_history = v_history,\n",
      "                                td_history = td_history,\n",
      "                                qmax_history = qmax_history,\n",
      "                                arguments=opt})\n",
      "        if opt.saveNetworkParams then\n",
      "            local nets = {network=w:clone():float()}\n",
      "            torch.save(filename..'.params.t7', nets, 'ascii')\n",
      "        end\n",
      "        deslim_agent()\n",
      "        print('Saved:', filename .. '.t7')\n",
      "        io.flush()\n",
      "        collectgarbage()\n",
      "    end\n",
      "end\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "Iteration ..\t0\t\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "Training Steps: \t10000\t\n"
       ]
      }
     ]
    }
   ],
   "metadata": {}
  }
 ]
}