rainbow_gcloud/Test_all.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rainbow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          Options\n",
      "                          seed: 123\n",
      "                          disable_cuda: False\n",
      "                          game: qbert\n",
      "                          T_max: 2000000\n",
      "                          max_episode_length: 108000\n",
      "                          history_length: 4\n",
      "                          hidden_size: 512\n",
      "                          noisy_std: 0.1\n",
      "                          atoms: 51\n",
      "                          V_min: -10\n",
      "                          V_max: 10\n",
      "                          model: None\n",
      "                          memory_capacity: 100000\n",
      "                          replay_frequency: 4\n",
      "                          priority_exponent: 0.5\n",
      "                          priority_weight: 0.4\n",
      "                          multi_step: 3\n",
      "                          discount: 0.99\n",
      "                          target_update: 8000\n",
      "                          reward_clip: 1\n",
      "                          lr: 6.25e-05\n",
      "                          adam_eps: 0.00015\n",
      "                          batch_size: 32\n",
      "                          learn_start: 20000\n",
      "                          evaluate: False\n",
      "                          evaluation_interval: 100000\n",
      "                          evaluation_episodes: 10\n",
      "                          evaluation_size: 500\n",
      "                          render: False\n",
      "                          enable_cudnn: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▌         | 100009/2000000 [33:33<211:13:23,  2.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-07T18:26:24] T = 100000 / 2000000 | Avg. reward: 147.5 | Avg. Q: 0.5167780474722385\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 200005/2000000 [1:14:08<672:07:08,  1.34s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-07T19:07:00] T = 200000 / 2000000 | Avg. reward: 177.5 | Avg. Q: 1.4737007971704006\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▌        | 300005/2000000 [1:55:08<374:42:17,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-07T19:48:00] T = 300000 / 2000000 | Avg. reward: 300.0 | Avg. Q: 2.6048021740317346\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 400006/2000000 [2:36:11<299:04:53,  1.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-07T20:29:02] T = 400000 / 2000000 | Avg. reward: 385.0 | Avg. Q: 3.459340757384896\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 500006/2000000 [3:17:21<192:49:50,  2.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-07T21:10:12] T = 500000 / 2000000 | Avg. reward: 602.5 | Avg. Q: 3.9480824509859085\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 600005/2000000 [3:58:34<346:19:42,  1.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-07T21:51:25] T = 600000 / 2000000 | Avg. reward: 635.0 | Avg. Q: 4.432433913946152\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 700005/2000000 [4:39:44<204:15:45,  1.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-07T22:32:35] T = 700000 / 2000000 | Avg. reward: 2075.0 | Avg. Q: 5.005757602512836\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 800005/2000000 [5:21:10<254:45:07,  1.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-07T23:14:01] T = 800000 / 2000000 | Avg. reward: 3137.5 | Avg. Q: 4.990583813521662\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 900006/2000000 [6:02:49<590:59:20,  1.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-07T23:55:40] T = 900000 / 2000000 | Avg. reward: 3122.5 | Avg. Q: 5.887928410529032\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 1000005/2000000 [6:44:07<270:21:12,  1.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-08T00:36:58] T = 1000000 / 2000000 | Avg. reward: 3290.0 | Avg. Q: 6.252039488071012\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 1100005/2000000 [7:25:54<310:45:00,  1.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-08T01:18:45] T = 1100000 / 2000000 | Avg. reward: 4840.0 | Avg. Q: 5.965679189138437\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 1200005/2000000 [8:07:55<402:14:05,  1.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-08T02:00:46] T = 1200000 / 2000000 | Avg. reward: 5067.5 | Avg. Q: 6.011825241173429\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▌   | 1300006/2000000 [8:49:57<333:56:32,  1.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-08T02:42:48] T = 1300000 / 2000000 | Avg. reward: 5885.0 | Avg. Q: 5.115982580621045\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 1400006/2000000 [9:32:05<297:36:44,  1.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-08T03:24:56] T = 1400000 / 2000000 | Avg. reward: 8005.0 | Avg. Q: 4.95420753755057\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 1500005/2000000 [10:14:29<188:09:57,  1.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-08T04:07:20] T = 1500000 / 2000000 | Avg. reward: 5602.5 | Avg. Q: 4.796778450507975\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 1600005/2000000 [10:55:54<889:48:26,  8.01s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-08T04:48:46] T = 1600000 / 2000000 | Avg. reward: 10782.5 | Avg. Q: 4.042200775198738\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▌ | 1700005/2000000 [11:36:47<165:14:31,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-08T05:29:38] T = 1700000 / 2000000 | Avg. reward: 10875.0 | Avg. Q: 3.9337151655120906\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 1800005/2000000 [12:18:08<108:08:17,  1.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-08T06:10:59] T = 1800000 / 2000000 | Avg. reward: 10440.0 | Avg. Q: 3.544111623888329\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▌| 1900005/2000000 [12:59:50<231:22:40,  8.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-08T06:52:41] T = 1900000 / 2000000 | Avg. reward: 12847.5 | Avg. Q: 3.5013383512719574\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2000000/2000000 [13:40:59<00:00, 40.60it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "29156"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%run Rainbow/main.py --game qbert --memory-capacity 100000 --T-max 2000000 --learn-start 20000 --target-update 8000 --batch-size 32\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp results results_main -rf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rainbow without prioritized memory replay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          Options\n",
      "                          seed: 123\n",
      "                          disable_cuda: False\n",
      "                          game: qbert\n",
      "                          T_max: 2000000\n",
      "                          max_episode_length: 108000\n",
      "                          history_length: 4\n",
      "                          hidden_size: 512\n",
      "                          noisy_std: 0.1\n",
      "                          atoms: 51\n",
      "                          V_min: -10\n",
      "                          V_max: 10\n",
      "                          model: None\n",
      "                          memory_capacity: 100000\n",
      "                          replay_frequency: 4\n",
      "                          priority_exponent: 0.0\n",
      "                          priority_weight: 0.4\n",
      "                          multi_step: 3\n",
      "                          discount: 0.99\n",
      "                          target_update: 8000\n",
      "                          reward_clip: 1\n",
      "                          lr: 6.25e-05\n",
      "                          adam_eps: 0.00015\n",
      "                          batch_size: 32\n",
      "                          learn_start: 20000\n",
      "                          evaluate: False\n",
      "                          evaluation_interval: 100000\n",
      "                          evaluation_episodes: 10\n",
      "                          evaluation_size: 500\n",
      "                          render: False\n",
      "                          enable_cudnn: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▌         | 100005/2000000 [30:52<208:04:44,  2.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T07:07:46] T = 100000 / 2000000 | Avg. reward: 417.5 | Avg. Q: 0.19993641841958742\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 200005/2000000 [1:07:56<285:15:22,  1.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T07:44:50] T = 200000 / 2000000 | Avg. reward: 427.5 | Avg. Q: 1.042030647989487\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▌        | 300005/2000000 [1:44:53<1664:29:26,  3.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T08:21:47] T = 300000 / 2000000 | Avg. reward: 530.0 | Avg. Q: 2.3141464616531393\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 400005/2000000 [2:21:49<303:00:59,  1.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T08:58:43] T = 400000 / 2000000 | Avg. reward: 2110.0 | Avg. Q: 3.744821659049099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 500005/2000000 [2:58:59<495:34:57,  1.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T09:35:52] T = 500000 / 2000000 | Avg. reward: 2187.5 | Avg. Q: 4.299369707170998\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 600005/2000000 [3:36:06<868:25:22,  2.23s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T10:12:59] T = 600000 / 2000000 | Avg. reward: 2957.5 | Avg. Q: 3.8988542787213842\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 700005/2000000 [4:13:18<642:55:06,  1.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T10:50:12] T = 700000 / 2000000 | Avg. reward: 3595.0 | Avg. Q: 4.40894180833825\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 800005/2000000 [4:50:24<308:31:53,  1.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T11:27:18] T = 800000 / 2000000 | Avg. reward: 3107.5 | Avg. Q: 5.32115873905629\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 900005/2000000 [5:27:38<325:18:18,  1.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T12:04:32] T = 900000 / 2000000 | Avg. reward: 4462.5 | Avg. Q: 5.512533653672666\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 1000005/2000000 [6:04:53<280:35:22,  1.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T12:41:47] T = 1000000 / 2000000 | Avg. reward: 4880.0 | Avg. Q: 6.092881500720673\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 1100005/2000000 [6:42:04<262:09:09,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T13:18:58] T = 1100000 / 2000000 | Avg. reward: 5557.5 | Avg. Q: 5.847063163867576\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 1200005/2000000 [7:19:25<234:02:08,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T13:56:19] T = 1200000 / 2000000 | Avg. reward: 9122.5 | Avg. Q: 5.965546770098347\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▌   | 1300005/2000000 [7:56:40<373:29:20,  1.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T14:33:34] T = 1300000 / 2000000 | Avg. reward: 10710.0 | Avg. Q: 6.421961995916521\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 1400005/2000000 [8:34:15<1477:23:44,  8.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T15:11:09] T = 1400000 / 2000000 | Avg. reward: 11442.5 | Avg. Q: 5.244469899771693\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 1500005/2000000 [9:11:43<136:50:48,  1.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T15:48:37] T = 1500000 / 2000000 | Avg. reward: 14012.5 | Avg. Q: 4.554135982565833\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 1600006/2000000 [9:49:12<291:48:47,  2.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T16:26:06] T = 1600000 / 2000000 | Avg. reward: 11345.0 | Avg. Q: 4.860578316185194\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▌ | 1700005/2000000 [10:26:44<97:02:38,  1.16s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T17:03:38] T = 1700000 / 2000000 | Avg. reward: 14790.0 | Avg. Q: 4.005488258914622\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 1800005/2000000 [11:04:25<141:16:56,  2.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T17:41:19] T = 1800000 / 2000000 | Avg. reward: 12202.5 | Avg. Q: 3.5466590311831867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▌| 1900005/2000000 [11:41:57<35:19:55,  1.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-09T18:18:51] T = 1900000 / 2000000 | Avg. reward: 15672.5 | Avg. Q: 3.619035056787965\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2000000/2000000 [12:18:44<00:00, 45.12it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "29156"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%run No_prioritize/main.py --game qbert --memory-capacity 100000 --T-max 2000000 --learn-start 20000 --target-update 8000 --batch-size 32 --priority-exponent 0.0\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp results results_no_prioritize -rf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rainbow without Dueling DQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          Options\n",
      "                          seed: 123\n",
      "                          disable_cuda: False\n",
      "                          game: qbert\n",
      "                          T_max: 2000000\n",
      "                          max_episode_length: 108000\n",
      "                          history_length: 4\n",
      "                          hidden_size: 512\n",
      "                          noisy_std: 0.1\n",
      "                          atoms: 51\n",
      "                          V_min: -10\n",
      "                          V_max: 10\n",
      "                          model: None\n",
      "                          memory_capacity: 100000\n",
      "                          replay_frequency: 4\n",
      "                          priority_exponent: 0.5\n",
      "                          priority_weight: 0.4\n",
      "                          multi_step: 3\n",
      "                          discount: 0.99\n",
      "                          target_update: 8000\n",
      "                          reward_clip: 1\n",
      "                          lr: 6.25e-05\n",
      "                          adam_eps: 0.00015\n",
      "                          batch_size: 32\n",
      "                          learn_start: 20000\n",
      "                          evaluate: False\n",
      "                          evaluation_interval: 100000\n",
      "                          evaluation_episodes: 10\n",
      "                          evaluation_size: 500\n",
      "                          render: False\n",
      "                          enable_cudnn: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▌         | 100006/2000000 [28:49<1248:56:09,  2.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T01:15:45] T = 100000 / 2000000 | Avg. reward: 3087.5 | Avg. Q: 0.5708649839162827\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 200007/2000000 [1:03:40<658:52:22,  1.32s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T01:50:36] T = 200000 / 2000000 | Avg. reward: 317.5 | Avg. Q: 1.8156143501996993\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▌        | 300007/2000000 [1:37:43<233:30:03,  2.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T02:24:39] T = 300000 / 2000000 | Avg. reward: 367.5 | Avg. Q: 3.0145425262451173\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 400007/2000000 [2:11:54<363:52:02,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T02:58:50] T = 400000 / 2000000 | Avg. reward: 357.5 | Avg. Q: 3.4631090483665465\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 500006/2000000 [2:45:48<153:16:48,  2.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T03:32:44] T = 500000 / 2000000 | Avg. reward: 295.0 | Avg. Q: 3.2986572308540345\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 600006/2000000 [3:19:50<204:33:21,  1.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T04:06:46] T = 600000 / 2000000 | Avg. reward: 437.5 | Avg. Q: 3.3482571959495546\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 700006/2000000 [3:54:29<257:27:13,  1.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T04:41:25] T = 700000 / 2000000 | Avg. reward: 495.0 | Avg. Q: 3.9653713116645815\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 800008/2000000 [4:30:06<346:06:11,  1.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T05:17:02] T = 800000 / 2000000 | Avg. reward: 525.0 | Avg. Q: 4.677910963058472\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 900006/2000000 [5:05:01<286:17:01,  1.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T05:51:57] T = 900000 / 2000000 | Avg. reward: 517.5 | Avg. Q: 5.219010433197021\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 1000006/2000000 [5:39:26<311:34:38,  1.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T06:26:22] T = 1000000 / 2000000 | Avg. reward: 2230.0 | Avg. Q: 5.4127912192344665\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 1100006/2000000 [6:14:46<120:14:59,  2.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T07:01:42] T = 1100000 / 2000000 | Avg. reward: 3987.5 | Avg. Q: 5.2638555140495304\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 1200007/2000000 [6:49:08<264:56:01,  1.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T07:36:04] T = 1200000 / 2000000 | Avg. reward: 4190.0 | Avg. Q: 6.139719071865081\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▌   | 1300001/2000000 [7:23:36<159:48:37,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T08:10:32] T = 1300000 / 2000000 | Avg. reward: 3302.5 | Avg. Q: 6.533256624221802\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 1400006/2000000 [7:58:12<186:34:33,  1.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T08:45:08] T = 1400000 / 2000000 | Avg. reward: 4072.5 | Avg. Q: 6.838954491615295\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 1500008/2000000 [8:32:24<169:47:07,  1.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T09:19:20] T = 1500000 / 2000000 | Avg. reward: 2985.0 | Avg. Q: 6.5132407603263855\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 1600007/2000000 [9:07:11<133:55:54,  1.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T09:54:07] T = 1600000 / 2000000 | Avg. reward: 3710.0 | Avg. Q: 7.147465727806091\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▌ | 1700007/2000000 [9:41:36<106:15:30,  1.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T10:28:32] T = 1700000 / 2000000 | Avg. reward: 4075.0 | Avg. Q: 7.3010306034088135\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 1800008/2000000 [10:16:00<38:52:20,  1.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T11:02:56] T = 1800000 / 2000000 | Avg. reward: 3710.0 | Avg. Q: 7.2867889423370364\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▌| 1900006/2000000 [10:50:28<24:27:29,  1.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T11:37:24] T = 1900000 / 2000000 | Avg. reward: 4187.5 | Avg. Q: 6.951221186637879\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2000000/2000000 [11:24:31<00:00, 48.70it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "359"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%run No_dueling/main.py --game qbert --memory-capacity 100000 --T-max 2000000 --learn-start 20000 --target-update 8000 --batch-size 32\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp results results_no_dueling -rf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rainbow without Lossy Net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          Options\n",
      "                          seed: 123\n",
      "                          disable_cuda: False\n",
      "                          game: qbert\n",
      "                          T_max: 2000000\n",
      "                          max_episode_length: 108000\n",
      "                          history_length: 4\n",
      "                          hidden_size: 512\n",
      "                          noisy_std: 0.0\n",
      "                          atoms: 51\n",
      "                          V_min: -10\n",
      "                          V_max: 10\n",
      "                          model: None\n",
      "                          memory_capacity: 100000\n",
      "                          replay_frequency: 4\n",
      "                          priority_exponent: 0.5\n",
      "                          priority_weight: 0.4\n",
      "                          multi_step: 3\n",
      "                          discount: 0.99\n",
      "                          target_update: 8000\n",
      "                          reward_clip: 1\n",
      "                          lr: 6.25e-05\n",
      "                          adam_eps: 0.00015\n",
      "                          batch_size: 32\n",
      "                          learn_start: 20000\n",
      "                          evaluate: False\n",
      "                          evaluation_interval: 100000\n",
      "                          evaluation_episodes: 10\n",
      "                          evaluation_size: 500\n",
      "                          render: False\n",
      "                          enable_cudnn: False\n",
      "                          eps_terminal: 0.001\n",
      "                          eps_dec: 0.999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▌         | 100005/2000000 [35:46<218:48:07,  2.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T23:04:57] T = 100000 / 2000000 | Avg. reward: 180.0 | Avg. Q: 0.8782453520074487\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 200005/2000000 [1:18:45<238:10:55,  2.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-10T23:47:56] T = 200000 / 2000000 | Avg. reward: 325.0 | Avg. Q: 2.1759650407135487\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▌        | 300005/2000000 [2:02:51<468:39:37,  1.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T00:32:02] T = 300000 / 2000000 | Avg. reward: 560.0 | Avg. Q: 2.7026800305577927\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 400005/2000000 [2:46:06<402:09:41,  1.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T01:15:17] T = 400000 / 2000000 | Avg. reward: 757.5 | Avg. Q: 3.426252356382087\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 500005/2000000 [3:29:22<340:49:15,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T01:58:33] T = 500000 / 2000000 | Avg. reward: 3710.0 | Avg. Q: 4.07745266562351\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 600005/2000000 [4:12:44<474:06:47,  1.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T02:41:56] T = 600000 / 2000000 | Avg. reward: 3090.0 | Avg. Q: 4.392951556461747\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 700005/2000000 [4:56:19<456:27:14,  1.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T03:25:31] T = 700000 / 2000000 | Avg. reward: 4212.5 | Avg. Q: 5.1034313074731905\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 800005/2000000 [5:39:57<525:50:02,  1.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T04:09:08] T = 800000 / 2000000 | Avg. reward: 4740.0 | Avg. Q: 6.094575451146055\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 900005/2000000 [6:22:55<459:31:50,  1.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T04:52:06] T = 900000 / 2000000 | Avg. reward: 4027.5 | Avg. Q: 5.589971600437282\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 1000005/2000000 [7:06:38<451:43:59,  1.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T05:35:50] T = 1000000 / 2000000 | Avg. reward: 5557.5 | Avg. Q: 5.143229668647458\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 1100005/2000000 [7:50:21<343:32:40,  1.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T06:19:32] T = 1100000 / 2000000 | Avg. reward: 4472.5 | Avg. Q: 4.972682084899883\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 1200005/2000000 [8:35:14<635:15:01,  2.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T07:04:25] T = 1200000 / 2000000 | Avg. reward: 8667.5 | Avg. Q: 4.912752358487279\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▌   | 1300005/2000000 [9:19:29<340:56:34,  1.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T07:48:40] T = 1300000 / 2000000 | Avg. reward: 6277.5 | Avg. Q: 5.182347442421738\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 1400005/2000000 [10:02:51<281:01:39,  1.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T08:32:02] T = 1400000 / 2000000 | Avg. reward: 8900.0 | Avg. Q: 4.816335923029761\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 1500005/2000000 [10:45:23<342:38:21,  2.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T09:14:34] T = 1500000 / 2000000 | Avg. reward: 10435.0 | Avg. Q: 4.69089500219969\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 1600005/2000000 [11:27:45<180:55:12,  1.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T09:56:56] T = 1600000 / 2000000 | Avg. reward: 13357.5 | Avg. Q: 4.321910565186553\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▌ | 1700005/2000000 [12:10:21<642:29:51,  7.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T10:39:32] T = 1700000 / 2000000 | Avg. reward: 11345.0 | Avg. Q: 4.142798460637784\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 1800005/2000000 [12:53:18<125:04:22,  2.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T11:22:29] T = 1800000 / 2000000 | Avg. reward: 15070.0 | Avg. Q: 3.8703726455648595\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▌| 1900005/2000000 [13:38:22<120:32:45,  4.34s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-11T12:07:33] T = 1900000 / 2000000 | Avg. reward: 16645.0 | Avg. Q: 3.5824031038385\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2000000/2000000 [14:26:25<00:00, 38.47it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "29161"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%run No_lossy/main.py --game qbert --memory-capacity 100000 --T-max 2000000 --learn-start 20000\\\n",
    "                      --target-update 8000 --batch-size 32 --noisy-std 0.0 --eps-dec 0.999 --eps-terminal 0.001\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp results results_no_lossy -rf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# C51 to C5 distributional learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          Options\n",
      "                          seed: 123\n",
      "                          disable_cuda: False\n",
      "                          game: qbert\n",
      "                          T_max: 2000000\n",
      "                          max_episode_length: 108000\n",
      "                          history_length: 4\n",
      "                          hidden_size: 512\n",
      "                          noisy_std: 0.1\n",
      "                          atoms: 5\n",
      "                          V_min: -10\n",
      "                          V_max: 10\n",
      "                          model: None\n",
      "                          memory_capacity: 100000\n",
      "                          replay_frequency: 4\n",
      "                          priority_exponent: 0.5\n",
      "                          priority_weight: 0.4\n",
      "                          multi_step: 3\n",
      "                          discount: 0.99\n",
      "                          target_update: 8000\n",
      "                          reward_clip: 1\n",
      "                          lr: 6.25e-05\n",
      "                          adam_eps: 0.00015\n",
      "                          batch_size: 32\n",
      "                          learn_start: 20000\n",
      "                          evaluate: False\n",
      "                          evaluation_interval: 100000\n",
      "                          evaluation_episodes: 10\n",
      "                          evaluation_size: 500\n",
      "                          render: False\n",
      "                          enable_cudnn: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▌         | 100005/2000000 [30:59<252:10:01,  2.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T04:05:16] T = 100000 / 2000000 | Avg. reward: 285.0 | Avg. Q: 0.64649067280069\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 200005/2000000 [1:08:39<284:50:43,  1.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T04:42:56] T = 200000 / 2000000 | Avg. reward: 397.5 | Avg. Q: 2.1107107226289807\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▌        | 300005/2000000 [1:46:04<232:59:22,  2.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T05:20:22] T = 300000 / 2000000 | Avg. reward: 732.5 | Avg. Q: 3.1826774521351324\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 400005/2000000 [2:24:05<212:07:24,  2.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T05:58:23] T = 400000 / 2000000 | Avg. reward: 817.5 | Avg. Q: 4.033103038231377\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 500005/2000000 [3:01:37<428:37:11,  1.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T06:35:54] T = 500000 / 2000000 | Avg. reward: 1432.5 | Avg. Q: 4.667721396876528\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 600005/2000000 [3:39:31<384:25:48,  1.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T07:13:49] T = 600000 / 2000000 | Avg. reward: 1102.5 | Avg. Q: 4.632157183305146\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 700005/2000000 [4:17:05<208:29:36,  1.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T07:51:22] T = 700000 / 2000000 | Avg. reward: 4190.0 | Avg. Q: 5.428130415668798\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 800005/2000000 [4:54:50<1383:12:49,  4.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T08:29:08] T = 800000 / 2000000 | Avg. reward: 4272.5 | Avg. Q: 5.846854645247571\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 900005/2000000 [5:32:33<422:34:15,  1.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T09:06:50] T = 900000 / 2000000 | Avg. reward: 4037.5 | Avg. Q: 5.717238048929648\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 1000005/2000000 [6:10:28<454:00:13,  1.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T09:44:45] T = 1000000 / 2000000 | Avg. reward: 4367.5 | Avg. Q: 6.731993937294697\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 1100005/2000000 [6:48:33<412:06:45,  1.65s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T10:22:51] T = 1100000 / 2000000 | Avg. reward: 4190.0 | Avg. Q: 6.678643137276187\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 1200005/2000000 [7:26:43<188:23:17,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T11:01:01] T = 1200000 / 2000000 | Avg. reward: 5387.5 | Avg. Q: 7.045241707290957\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▌   | 1300005/2000000 [8:04:57<204:23:24,  1.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T11:39:14] T = 1300000 / 2000000 | Avg. reward: 6155.0 | Avg. Q: 6.242279496017896\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 1400005/2000000 [8:43:13<440:47:02,  2.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T12:17:31] T = 1400000 / 2000000 | Avg. reward: 6870.0 | Avg. Q: 6.4203018138698225\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 1500005/2000000 [9:21:30<760:21:30,  5.47s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T12:55:47] T = 1500000 / 2000000 | Avg. reward: 8647.5 | Avg. Q: 4.941119944217108\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 1600005/2000000 [9:59:30<230:11:19,  2.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T13:33:47] T = 1600000 / 2000000 | Avg. reward: 10712.5 | Avg. Q: 4.488438018770644\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▌ | 1700005/2000000 [10:37:25<177:06:54,  2.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T14:11:43] T = 1700000 / 2000000 | Avg. reward: 14025.0 | Avg. Q: 4.68483643929265\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 1800005/2000000 [11:15:44<81:44:56,  1.47s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T14:50:01] T = 1800000 / 2000000 | Avg. reward: 13200.0 | Avg. Q: 3.7319132055239344\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▌| 1900005/2000000 [11:53:43<24:36:27,  1.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-06-12T15:28:01] T = 1900000 / 2000000 | Avg. reward: 11207.5 | Avg. Q: 3.7564281344478085\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2000000/2000000 [12:31:02<00:00, 44.38it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "29156"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%run Rainbow/main.py --game qbert --memory-capacity 100000 --T-max 2000000 --learn-start 20000 --target-update 8000 --batch-size 32 --atoms 5\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp results results_c5 -rf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (gym)",
   "language": "python",
   "name": "gym"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}