tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RL4RS Tutorial v1.0 2022.07.13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 1: Installing\n",
    "# !git clone https://github.com/fuxiAIlab/RL4RS\n",
    "# !export PYTHONPATH=$PYTHONPATH:`pwd`/rl4rs\n",
    "# !conda env create -f environment.yml\n",
    "# !conda activate rl4rs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 2: Dataset Introduction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](https://i.bmp.ovh/imgs/2022/07/14/c5e51bb495e704ec.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "timestamp@session_id@sequence_id@exposed_items@user_feedback@user_seqfeature@user_protrait@item_feature@behavior_policy_id\r\n",
      "2992008@1@1@3,5,29,72,53,52,164,211,172@1,1,1,1,1,1,1,1,1@24,7,1,127,74,47,212,199,6,15,10,127,126,76,220,196,172,196,15,39,31,132,80,61,200,219,1,5,14,101,40,52,235,238,164,1,14,20,77,80,40,239,233,164,164,33,31,14,139,83,83,125,184,240,160,8,1,25,51,76,43,235,211,164,14,4,9,126,116,43,164,213,183,6,14,4,43,126,57,183,188,164,10,39,25,102,109,111,160,160,157,183,10,5,32,43,103,50,242,172,171,171,35,9,5,86,48,88,218,236,215@64054,50887,66367,44932,59460,20543,83978,50138,74820,58670,3.146,13.82,1.268,4.848,7.523,9.176,9.875,10.66,9.355,9.774,10.29,17.51,7.418,6.973,0,2.683,0,17.45,2.368,17.24,12.07,0,0,0,0,12.02,10.32,11.78,1.009,30.16,0,0@-0.2137,2.0783,-0.3633,-0.142,1.372,-0.6294,-1.8973,-1.7015,-0.8995,0.4999,-1.0659,-1.1485,1.4111,1.5868,0.2704,1.5112,-0.4576,1.2751,1.0225,1.1452,-1.4702,-1.388,0.6215,1.5247,1.6815,0.6815,1.3391,-0.7501,1.0955,-1.1044,0.6773,-1.3632,0.1231,0.805,-0.3372,0.5489,-0.445,-1.5474,-0.8624,1.201;-0.2137,-0.7579,-0.3633,-0.1352,1.3293,-0.7099,-1.8973,-1.677,1.6924,1.1324,-0.1024,1.1928,1.7338,-0.2146,0.6337,-1.0009,-1.2873,-0.6144,-1.6587,0.4634,1.0691,1.2389,0.7369,0.9045,-1.1352,-1.3375,1.7778,-1.1019,-1.6901,1.1034,-0.5661,-0.8314,0.0778,-0.7026,-1.7152,-0.5689,0.1662,-0.505,0.3174,1.4195;1.5653,-0.0489,-0.3633,-0.1337,1.7984,-0.4087,-1.8973,-1.3832,0.3859,1.6068,0.6345,1.5253,0.0213,1.329,-0.658,0.3951,1.7631,-1.2467,0.4961,1.1013,1.2305,1.6542,-0.4394,-0.5117,1.6133,0.994,1.2942,0.664,-1.4112,-0.222,0.8467,1.3883,-1.7096,-0.5375,0.3698,1.1702,1.3394,0.3454,-1.5845,-1.6395;-0.2137,-0.4034,-0.3633,-0.1419,1.2634,-0.5603,-0.4843,-0.8568,-0.624,0.3182,-1.0695,1.4773,-0.6808,-1.5554,1.2291,1.5043,-0.496,-0.7835,0.5562,1.3106,1.4273,1.7788,0.0333,-0.2016,-0.2801,-1.0671,1.4116,0.2349,-0.833,-0.3753,-1.3522,-1.307,1.5376,1.7131,-1.2954,-1.1362,-0.5699,1.176,-1.4218,0.2777;-1.1033,-0.7579,-0.3633,-0.1447,0.4765,-0.8246,-0.4843,-1.0894,0.0892,-1.1184,1.0839,0.4249,0.0177,1.494,-0.5806,1.1035,0.8532,1.7787,0.4697,0.7334,1.0433,0.6194,0.2678,0.9217,-0.0502,-0.702,0.8589,1.5998,-0.4045,0.0029,0.1793,0.1596,1.3495,-1.6538,0.6604,1.5552,-1.3554,0.448,-0.4895,-0.2792;-1.1033,-0.7579,-0.3633,-0.1454,0.3486,-0.8251,-0.4843,-1.1017,0.4282,-1.3875,1.4182,1.1448,-1.6062,1.0677,-0.8564,0.4228,-0.3461,-1.1989,-0.7449,-1.1872,1.1756,0.8443,-1.4354,-1.3835,-1.3041,-0.6809,1.3356,0.7098,-1.5541,0.6127,-1.3251,1.5667,-0.194,0.3096,0.5743,0.3632,0.4685,0.2031,-0.2352,-0.3144;-1.1033,1.3692,-0.3633,-0.1264,0.2788,0.8519,0.9287,0.2693,0.227,1.6404,-0.3505,-0.7714,-1.6984,-0.9709,-0.1736,-1.3879,-1.2559,-1.6143,-0.9292,-0.7957,1.5112,-1.4711,1.289,1.4627,-1.3436,1.5944,0.0644,1.4098,-1.2242,-1.0362,-1.2132,0.4403,-1.3786,1.7095,0.804,1.1128,1.2277,0.4315,1.5446,0.8169;0.6758,-0.4034,-0.3633,-0.144,0.0152,-0.418,0.9287,0.8446,-0.4404,-0.9199,0.7064,-1.5187,-1.3828,0.9783,0.1897,-1.3084,-1.1409,-1.5812,1.5904,1.2566,-0.1473,-1.6164,-1.5003,-0.8046,1.0959,1.0115,1.4151,-1.2673,-1.0915,-0.4673,1.0534,-0.5935,0.5272,0.7296,0.6712,1.2884,-0.4253,-0.2436,0.7581,-1.6677;-1.1033,1.3692,-0.3633,-0.1342,-0.5042,2.4328,0.9287,0.3672,1.3816,0.5907,0.5051,-1.0662,1.5671,0.4592,-0.7992,-1.3222,0.1211,-0.7357,-1.6813,-1.2277,-0.1247,-0.4916,-1.6122,1.3455,-1.6382,1.6927,0.6309,-1.2884,0.82,-0.2186,0.8332,-1.3599,-1.302,-0.5375,1.4607,1.1229,1.4413,1.5996,1.1378,0.0204@1\r\n"
     ]
    }
   ],
   "source": [
    "! head -2 /project/wangkai/rl4rs_benchmark_materials/raw_data/rl4rs_dataset_a_sl.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of records:  937949\n",
      "number of users:  937949\n",
      "number of items per page:  9\n",
      "avg. lengths of user history:  36.35307676643399\n",
      "avg. dimensions of user_protrait:  42.0\n",
      "avg. dimensions of item_feature:  352.0\n",
      "number of behavior_policy_id:  1\n"
     ]
    }
   ],
   "source": [
    "with open('/project/wangkai/rl4rs_benchmark_materials/raw_data/rl4rs_dataset_a_sl.csv', 'r') as f:\n",
    "    data = f.read().split('\\n')[1:-1]\n",
    "    print('number of records: ', len(data))\n",
    "    print('number of users: ', len(set([x.split('@')[1] for x in data])))\n",
    "    print('number of items per page: ', len(data[0].split('@')[3].split(',')))\n",
    "    print('avg. lengths of user history: ', sum([len(x.split('@')[5].split(',')) for x in data])/len(data))\n",
    "    print('avg. dimensions of user_protrait: ', sum([len(x.split('@')[6].split(',')) for x in data])/len(data))\n",
    "    print('avg. dimensions of item_feature: ', sum([len(x.split('@')[7].split(',')) for x in data])/len(data))\n",
    "    print('number of behavior_policy_id: ', len(set([x.split('@')[-1] for x in data])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 2: Data PreProcess\n",
    "# See https://github.com/fuxiAIlab/RL4RS/blob/main/reproductions/run_split.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 3: Simulation Environment (Local)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys\n",
    "import gym\n",
    "import numpy as np\n",
    "from copy import deepcopy\n",
    "import tensorflow as tf\n",
    "from rl4rs.utils.datautil import FeatureUtil\n",
    "from rl4rs.env.slate import SlateRecEnv, SlateState\n",
    "from rl4rs.env.seqslate import SeqSlateRecEnv, SeqSlateState"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /project/wangkai/rl4rs_benchmark_materials/simulator/finetuned/simulator_a_dien/model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /project/wangkai/rl4rs_benchmark_materials/simulator/finetuned/simulator_a_dien/model\n"
     ]
    }
   ],
   "source": [
    "config = {\"epoch\": 10000, \"maxlen\": 64, \"batch_size\": 64, \"action_size\": 284, \n",
    "          \"class_num\": 2, \"dense_feature_num\": 432, \"category_feature_num\": 21, \n",
    "          \"category_hash_size\": 100000, \"seq_num\": 2, \"emb_size\": 128, \"is_eval\": False,\n",
    "          \"hidden_units\": 128, \"max_steps\": 9, \"action_emb_size\": 32,\n",
    "          \"sample_file\": '/project/wangkai/rl4rs_benchmark_materials/simulator/rl4rs_dataset_a_shuf.csv', \n",
    "          \"model_file\": \"/project/wangkai/rl4rs_benchmark_materials/simulator/finetuned/simulator_a_dien/model\",\n",
    "          \"iteminfo_file\": '/project/wangkai/rl4rs_benchmark_materials/raw_data/item_info.csv', \n",
    "          \"support_rllib_mask\": True, 'env': \"SlateRecEnv-v0\"}\n",
    "\n",
    "sim = SlateRecEnv(config, state_cls=SlateState)\n",
    "env = gym.make('SlateRecEnv-v0', recsim=sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batchsize of batched environment:  64\n",
      "step:  0  action 31  reward:  0  offline reward:  0  done:  0\n",
      "step:  1  action 28  reward:  0  offline reward:  0  done:  0\n",
      "step:  2  action 20  reward:  0  offline reward:  0  done:  0\n",
      "step:  3  action 87  reward:  0  offline reward:  0  done:  0\n",
      "step:  4  action 73  reward:  0  offline reward:  0  done:  0\n",
      "step:  5  action 146  reward:  0  offline reward:  0  done:  0\n",
      "step:  6  action 235  reward:  0  offline reward:  0  done:  0\n",
      "step:  7  action 233  reward:  0  offline reward:  0  done:  0\n",
      "step:  8  action 166  reward:  130.2745725877583  offline reward:  118.5  done:  1\n",
      "observation type:  <class 'dict'>\n",
      "size of obs.action_mask:  284\n",
      "size of obs.obs:  256\n"
     ]
    }
   ],
   "source": [
    "obs = env.reset()\n",
    "print('batchsize of batched environment: ', len(obs))\n",
    "\n",
    "for i in range(config[\"max_steps\"]):\n",
    "    action = env.offline_action\n",
    "    next_obs, reward, done, info = env.step(action)\n",
    "    print('step: ', i, ' action', action[0], ' reward: ', reward[0], ' offline reward: ', env.offline_reward[0], ' done: ', done[0])\n",
    "    \n",
    "print('observation type: ', type(next_obs[0]))\n",
    "print('size of obs.action_mask: ', len(next_obs[0]['action_mask']))\n",
    "print('size of obs.obs: ', len(next_obs[0]['obs']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Env with continus action space (without item mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /project/wangkai/rl4rs_benchmark_materials/simulator/finetuned/simulator_a_dien/model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /project/wangkai/rl4rs_benchmark_materials/simulator/finetuned/simulator_a_dien/model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "size of action embedding  (284, 32)\n",
      "step:  0  action 53  reward:  0  done:  0\n",
      "step:  1  action 53  reward:  0  done:  0\n",
      "step:  2  action 53  reward:  0  done:  0\n",
      "step:  3  action 53  reward:  0  done:  0\n",
      "step:  4  action 53  reward:  0  done:  0\n",
      "step:  5  action 53  reward:  0  done:  0\n",
      "step:  6  action 53  reward:  0  done:  0\n",
      "step:  7  action 53  reward:  0  done:  0\n",
      "step:  8  action 53  reward:  47.939698447287086  done:  1\n"
     ]
    }
   ],
   "source": [
    "config_conti = deepcopy(config)\n",
    "config_conti['support_conti_env'] = True\n",
    "config_conti['support_rllib_mask'] = False\n",
    "sim = SlateRecEnv(config_conti, state_cls=SlateState)\n",
    "env = gym.make('SlateRecEnv-v0', recsim=sim)\n",
    "obs = env.reset()\n",
    "action_vec = np.full((batch_size, 32), 1)\n",
    "print('size of action embedding ', np.array(env.samples.action_emb).shape)\n",
    "for i in range(config[\"max_steps\"]):\n",
    "    next_obs, reward, done, info = env.step(action_vec)\n",
    "    action = SlateState.get_nearest_neighbor(action_vec, env.samples.action_emb)\n",
    "    print('step: ', i, ' action', action[0], ' reward: ', reward[0], ' done: ', done[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Env with continus action space (with item mask)\n",
    "# See https://github.com/fuxiAIlab/RL4RS/blob/main/script/modelfree_train.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Take raw features as observation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /project/wangkai/rl4rs_benchmark_materials/simulator/finetuned/simulator_a_dien/model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /project/wangkai/rl4rs_benchmark_materials/simulator/finetuned/simulator_a_dien/model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "observation type:  <class 'dict'>\n",
      "size of obs.action_mask:  284\n",
      "size of obs.category_feature:  21\n",
      "size of obs.dense_feature:  432\n",
      "size of obs.sequence_feature:  (2,64)\n"
     ]
    }
   ],
   "source": [
    "from copy import deepcopy\n",
    "config_rawstate = deepcopy(config)\n",
    "config_rawstate['support_rllib_mask'] = True\n",
    "config_rawstate['support_conti_env'] = False\n",
    "config_rawstate['rawstate_as_obs'] = True\n",
    "sim = SlateRecEnv(config_rawstate, state_cls=SlateState)\n",
    "env = gym.make('SlateRecEnv-v0', recsim=sim)\n",
    "obs = env.reset()\n",
    "print('observation type: ', type(obs[0]))\n",
    "print('size of obs.action_mask: ', len(obs[0]['action_mask']))\n",
    "print('size of obs.category_feature: ', len(obs[0]['category_feature']))\n",
    "print('size of obs.dense_feature: ', len(obs[0]['dense_feature']))\n",
    "print('size of obs.sequence_feature: ', obs[0]['sequence_feature'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 3: Simulation Environment (Remote)\n",
    "# start http-based Env, then run RLlib library\n",
    "# nohup python -u rl4rs/server/gymHttpServer.py &\n",
    "# bash run_modelfree_rl.sh DQN/PPO/DDPG/PG/PG_conti/etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 4: Model-free Training (RLLib)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/project/miniconda3/envs/rl4rs/lib/python3.6/site-packages/ray/autoscaler/_private/cli_logger.py:61: FutureWarning: Not all Ray CLI dependencies were found. In Ray 1.4+, the Ray CLI, autoscaler, and dashboard will only be usable via `pip install 'ray[default]'`. Please update your install command.\n",
      "  \"update your install command.\", FutureWarning)\n",
      "2022-07-14 17:34:22,072\tWARNING utils.py:511 -- Detecting docker specified CPUs. In previous versions of Ray, CPU detection in containers was incorrect. Please ensure that Ray has enough CPUs allocated. As a temporary workaround to revert to the prior behavior, set `RAY_USE_MULTIPROCESSING_CPU_COUNT=1` as an env var before starting Ray. Set the env var: `RAY_DISABLE_DOCKER_CPU_WARNING=1` to mute this warning.\n",
      "2022-07-14 17:34:45,283\tERROR services.py:1254 -- Failed to start the dashboard: Failed to start the dashboard. The last 10 lines of /tmp/ray/session_2022-07-14_17-34-22_052438_47614/logs/dashboard.log:\n",
      "2022-07-14 17:34:41,447\tINFO dashboard.py:92 -- Setup static dir for dashboard: /project/miniconda3/envs/rl4rs/lib/python3.6/site-packages/ray/new_dashboard/client/build\n",
      "2022-07-14 17:34:41,451\tINFO head.py:82 -- Connect to GCS at b'192.168.98.96:43889'\n",
      "2022-07-14 17:34:41,453\tINFO utils.py:202 -- Get all modules by type: DashboardHeadModule\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'epoch': 2, 'maxlen': 64, 'batch_size': 64, 'action_size': 284, 'class_num': 2, 'dense_feature_num': 432, 'category_feature_num': 21, 'category_hash_size': 100000, 'seq_num': 2, 'emb_size': 128, 'is_eval': False, 'hidden_units': 128, 'max_steps': 9, 'action_emb_size': 32, 'sample_file': '/project/wangkai/rl4rs_benchmark_materials/simulator/rl4rs_dataset_a_shuf.csv', 'model_file': '/project/wangkai/rl4rs_benchmark_materials/simulator/finetuned/simulator_a_dien/model', 'iteminfo_file': '/project/wangkai/rl4rs_benchmark_materials/raw_data/item_info.csv', 'remote_base': 'http://127.0.0.1:5000', 'trial_name': 'all', 'support_rllib_mask': True, 'env': 'SlateRecEnv-v0'}\n",
      "rllib_config {'env': 'rllibEnv-v0', 'gamma': 1, 'explore': True, 'exploration_config': {'type': 'SoftQ'}, 'num_gpus': 1, 'num_workers': 2, 'framework': 'tf', 'rollout_fragment_length': 9, 'batch_mode': 'complete_episodes', 'train_batch_size': 576, 'evaluation_interval': 1, 'evaluation_num_episodes': 8192, 'evaluation_config': {'explore': False}, 'log_level': 'INFO', 'use_critic': True, 'use_gae': True, 'lambda': 1.0, 'kl_coeff': 0.2, 'sgd_minibatch_size': 256, 'shuffle_sequences': True, 'num_sgd_iter': 1, 'lr': 0.0001, 'vf_loss_coeff': 0.5, 'clip_param': 0.3, 'vf_clip_param': 500.0, 'kl_target': 0.01}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-14 17:34:48,364\tINFO trainer.py:706 -- Tip: set framework=tfe or the --eager flag to enable TensorFlow eager execution\n",
      "\u001b[2m\u001b[36m(pid=47823)\u001b[0m /project/miniconda3/envs/rl4rs/lib/python3.6/site-packages/gym/logger.py:34: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "\u001b[2m\u001b[36m(pid=47823)\u001b[0m   warnings.warn(colorize(\"%s: %s\" % (\"WARN\", msg % args), \"yellow\"))\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m /project/miniconda3/envs/rl4rs/lib/python3.6/site-packages/gym/logger.py:34: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   warnings.warn(colorize(\"%s: %s\" % (\"WARN\", msg % args), \"yellow\"))\n",
      "\u001b[2m\u001b[36m(pid=47823)\u001b[0m WARNING:tensorflow:From /project/miniconda3/envs/rl4rs/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "\u001b[2m\u001b[36m(pid=47823)\u001b[0m Instructions for updating:\n",
      "\u001b[2m\u001b[36m(pid=47823)\u001b[0m If using Keras pass *_constraint arguments to layers.\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m WARNING:tensorflow:From /project/miniconda3/envs/rl4rs/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m Instructions for updating:\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m If using Keras pass *_constraint arguments to layers.\n",
      "\u001b[2m\u001b[36m(pid=47823)\u001b[0m 2022-07-14 17:35:04,153\tINFO dynamic_tf_policy.py:472 -- Testing `compute_actions` w/ dummy batch.\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m 2022-07-14 17:35:04,217\tINFO dynamic_tf_policy.py:472 -- Testing `compute_actions` w/ dummy batch.\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m 2022-07-14 17:35:04,217\tINFO tf_run_builder.py:87 -- Executing TF run without tracing. To dump TF timeline traces to disk, set the TF_TIMELINE_DIR environment variable.\n",
      "\u001b[2m\u001b[36m(pid=47823)\u001b[0m 2022-07-14 17:35:04,221\tINFO dynamic_tf_policy.py:481 -- Adding extra-action-fetch `action_prob` to view-reqs.\n",
      "\u001b[2m\u001b[36m(pid=47823)\u001b[0m 2022-07-14 17:35:04,222\tINFO dynamic_tf_policy.py:481 -- Adding extra-action-fetch `action_logp` to view-reqs.\n",
      "\u001b[2m\u001b[36m(pid=47823)\u001b[0m 2022-07-14 17:35:04,223\tINFO dynamic_tf_policy.py:481 -- Adding extra-action-fetch `action_dist_inputs` to view-reqs.\n",
      "\u001b[2m\u001b[36m(pid=47823)\u001b[0m 2022-07-14 17:35:04,223\tINFO dynamic_tf_policy.py:481 -- Adding extra-action-fetch `vf_preds` to view-reqs.\n",
      "\u001b[2m\u001b[36m(pid=47823)\u001b[0m 2022-07-14 17:35:04,223\tINFO dynamic_tf_policy.py:488 -- Testing `postprocess_trajectory` w/ dummy batch.\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m 2022-07-14 17:35:04,283\tINFO dynamic_tf_policy.py:481 -- Adding extra-action-fetch `action_prob` to view-reqs.\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m 2022-07-14 17:35:04,284\tINFO dynamic_tf_policy.py:481 -- Adding extra-action-fetch `action_logp` to view-reqs.\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m 2022-07-14 17:35:04,285\tINFO dynamic_tf_policy.py:481 -- Adding extra-action-fetch `action_dist_inputs` to view-reqs.\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m 2022-07-14 17:35:04,286\tINFO dynamic_tf_policy.py:481 -- Adding extra-action-fetch `vf_preds` to view-reqs.\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m 2022-07-14 17:35:04,286\tINFO dynamic_tf_policy.py:488 -- Testing `postprocess_trajectory` w/ dummy batch.\n",
      "\u001b[2m\u001b[36m(pid=47823)\u001b[0m WARNING:tensorflow:From /project/miniconda3/envs/rl4rs/lib/python3.6/site-packages/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "\u001b[2m\u001b[36m(pid=47823)\u001b[0m Instructions for updating:\n",
      "\u001b[2m\u001b[36m(pid=47823)\u001b[0m Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m WARNING:tensorflow:From /project/miniconda3/envs/rl4rs/lib/python3.6/site-packages/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m Instructions for updating:\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /project/miniconda3/envs/rl4rs/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-14 17:35:06,254\tINFO dynamic_tf_policy.py:472 -- Testing `compute_actions` w/ dummy batch.\n",
      "2022-07-14 17:35:06,257\tINFO tf_run_builder.py:87 -- Executing TF run without tracing. To dump TF timeline traces to disk, set the TF_TIMELINE_DIR environment variable.\n",
      "2022-07-14 17:35:06,467\tINFO dynamic_tf_policy.py:481 -- Adding extra-action-fetch `action_prob` to view-reqs.\n",
      "2022-07-14 17:35:06,470\tINFO dynamic_tf_policy.py:481 -- Adding extra-action-fetch `action_logp` to view-reqs.\n",
      "2022-07-14 17:35:06,472\tINFO dynamic_tf_policy.py:481 -- Adding extra-action-fetch `action_dist_inputs` to view-reqs.\n",
      "2022-07-14 17:35:06,474\tINFO dynamic_tf_policy.py:481 -- Adding extra-action-fetch `vf_preds` to view-reqs.\n",
      "2022-07-14 17:35:06,475\tINFO dynamic_tf_policy.py:488 -- Testing `postprocess_trajectory` w/ dummy batch.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /project/miniconda3/envs/rl4rs/lib/python3.6/site-packages/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-14 17:35:07,609\tINFO rollout_worker.py:1344 -- Built policy map: {'default_policy': <ray.rllib.policy.tf_policy_template.PPOTFPolicy object at 0x7f521435d748>}\n",
      "2022-07-14 17:35:07,610\tINFO rollout_worker.py:1345 -- Built preprocessor map: {'default_policy': <ray.rllib.models.preprocessors.DictFlatteningPreprocessor object at 0x7f521435d320>}\n",
      "2022-07-14 17:35:07,611\tINFO rollout_worker.py:602 -- Built filter map: {'default_policy': <ray.rllib.utils.filter.NoFilter object at 0x7f51f44d3e80>}\n",
      "2022-07-14 17:35:10,276\tWARNING deprecation.py:34 -- DeprecationWarning: `simple_optimizer` has been deprecated. This will raise an error in the future!\n",
      "/project/miniconda3/envs/rl4rs/lib/python3.6/site-packages/gym/logger.py:34: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize(\"%s: %s\" % (\"WARN\", msg % args), \"yellow\"))\n",
      "2022-07-14 17:35:13,350\tINFO dynamic_tf_policy.py:472 -- Testing `compute_actions` w/ dummy batch.\n",
      "2022-07-14 17:35:13,405\tINFO dynamic_tf_policy.py:481 -- Adding extra-action-fetch `action_prob` to view-reqs.\n",
      "2022-07-14 17:35:13,407\tINFO dynamic_tf_policy.py:481 -- Adding extra-action-fetch `action_logp` to view-reqs.\n",
      "2022-07-14 17:35:13,409\tINFO dynamic_tf_policy.py:481 -- Adding extra-action-fetch `action_dist_inputs` to view-reqs.\n",
      "2022-07-14 17:35:13,410\tINFO dynamic_tf_policy.py:481 -- Adding extra-action-fetch `vf_preds` to view-reqs.\n",
      "2022-07-14 17:35:13,411\tINFO dynamic_tf_policy.py:488 -- Testing `postprocess_trajectory` w/ dummy batch.\n",
      "2022-07-14 17:35:14,372\tINFO rollout_worker.py:1344 -- Built policy map: {'default_policy': <ray.rllib.policy.tf_policy_template.PPOTFPolicy object at 0x7f4e44101b70>}\n",
      "2022-07-14 17:35:14,373\tINFO rollout_worker.py:1345 -- Built preprocessor map: {'default_policy': <ray.rllib.models.preprocessors.DictFlatteningPreprocessor object at 0x7f4e44101470>}\n",
      "2022-07-14 17:35:14,374\tINFO rollout_worker.py:602 -- Built filter map: {'default_policy': <ray.rllib.utils.filter.NoFilter object at 0x7f4e44062fd0>}\n",
      "2022-07-14 17:35:14,375\tINFO trainable.py:109 -- Trainable.setup took 26.013 seconds. If your trainable is slow to initialize, consider setting reuse_actors=True to reduce actor creation overheads.\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m 2022-07-14 17:35:14,443\tINFO rollout_worker.py:737 -- Generating sample batch of size 9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainer_default_config {'num_workers': 2, 'num_envs_per_worker': 1, 'create_env_on_driver': False, 'rollout_fragment_length': 200, 'batch_mode': 'truncate_episodes', 'gamma': 0.99, 'lr': 5e-05, 'train_batch_size': 4000, 'model': {'_use_default_native_models': False, 'fcnet_hiddens': [256, 256], 'fcnet_activation': 'tanh', 'conv_filters': None, 'conv_activation': 'relu', 'post_fcnet_hiddens': [], 'post_fcnet_activation': 'relu', 'free_log_std': False, 'no_final_linear': False, 'vf_share_layers': False, 'use_lstm': False, 'max_seq_len': 20, 'lstm_cell_size': 256, 'lstm_use_prev_action': False, 'lstm_use_prev_reward': False, '_time_major': False, 'use_attention': False, 'attention_num_transformer_units': 1, 'attention_dim': 64, 'attention_num_heads': 1, 'attention_head_dim': 32, 'attention_memory_inference': 50, 'attention_memory_training': 50, 'attention_position_wise_mlp_dim': 32, 'attention_init_gru_gate_bias': 2.0, 'attention_use_n_prev_actions': 0, 'attention_use_n_prev_rewards': 0, 'num_framestacks': 'auto', 'dim': 84, 'grayscale': False, 'zero_mean': True, 'custom_model': None, 'custom_model_config': {}, 'custom_action_dist': None, 'custom_preprocessor': None, 'lstm_use_prev_action_reward': -1, 'framestack': True}, 'optimizer': {}, 'horizon': None, 'soft_horizon': False, 'no_done_at_end': False, 'env': None, 'observation_space': None, 'action_space': None, 'env_config': {}, 'env_task_fn': None, 'render_env': False, 'record_env': False, 'clip_rewards': None, 'normalize_actions': True, 'clip_actions': False, 'preprocessor_pref': 'deepmind', 'log_level': 'WARN', 'callbacks': <class 'ray.rllib.agents.callbacks.DefaultCallbacks'>, 'ignore_worker_failures': False, 'log_sys_usage': True, 'fake_sampler': False, 'framework': 'tf', 'eager_tracing': False, 'explore': True, 'exploration_config': {'type': 'StochasticSampling'}, 'evaluation_interval': None, 'evaluation_num_episodes': 10, 'evaluation_parallel_to_training': False, 'in_evaluation': False, 'evaluation_config': {}, 'evaluation_num_workers': 0, 'custom_eval_function': None, 'sample_async': False, 'sample_collector': <class 'ray.rllib.evaluation.collectors.simple_list_collector.SimpleListCollector'>, 'observation_filter': 'NoFilter', 'synchronize_filters': True, 'tf_session_args': {'intra_op_parallelism_threads': 2, 'inter_op_parallelism_threads': 2, 'gpu_options': {'allow_growth': True}, 'log_device_placement': False, 'device_count': {'CPU': 1}, 'allow_soft_placement': True}, 'local_tf_session_args': {'intra_op_parallelism_threads': 8, 'inter_op_parallelism_threads': 8}, 'compress_observations': False, 'collect_metrics_timeout': 180, 'metrics_smoothing_episodes': 100, 'remote_worker_envs': False, 'remote_env_batch_wait_ms': 0, 'min_iter_time_s': 0, 'timesteps_per_iteration': 0, 'seed': None, 'extra_python_environs_for_driver': {}, 'extra_python_environs_for_worker': {}, 'num_gpus': 0, '_fake_gpus': False, 'num_cpus_per_worker': 1, 'num_gpus_per_worker': 0, 'custom_resources_per_worker': {}, 'num_cpus_for_driver': 1, 'placement_strategy': 'PACK', 'input': 'sampler', 'input_config': {}, 'actions_in_input_normalized': False, 'input_evaluation': ['is', 'wis'], 'postprocess_inputs': False, 'shuffle_buffer_size': 0, 'output': None, 'output_compress_columns': ['obs', 'new_obs'], 'output_max_file_size': 67108864, 'multiagent': {'policies': {}, 'policy_mapping_fn': None, 'policies_to_train': None, 'observation_fn': None, 'replay_mode': 'independent', 'count_steps_by': 'env_steps'}, 'logger_config': None, 'simple_optimizer': -1, 'monitor': -1, 'use_critic': True, 'use_gae': True, 'lambda': 1.0, 'kl_coeff': 0.2, 'sgd_minibatch_size': 128, 'shuffle_sequences': True, 'num_sgd_iter': 30, 'lr_schedule': None, 'vf_loss_coeff': 1.0, 'entropy_coeff': 0.0, 'entropy_coeff_schedule': None, 'clip_param': 0.3, 'vf_clip_param': 10.0, 'grad_clip': None, 'kl_target': 0.01, 'vf_share_layers': -1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m 2022-07-14 17:35:14,667\tINFO sampler.py:593 -- Raw obs from env: { 0: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                    'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=14.742, mean=0.205)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   1: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                    'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=6.099, mean=-0.391)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   2: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                    'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=10.2, mean=0.211)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   3: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                    'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=14.113, mean=0.086)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   4: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                    'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=15.212, mean=0.354)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   5: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                    'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=13.614, mean=-0.035)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   6: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                    'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=13.649, mean=-0.024)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   7: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                    'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=16.506, mean=0.407)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   8: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                    'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=14.946, mean=0.133)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   9: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                    'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=12.712, mean=-0.026)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   10: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=10.496, mean=0.132)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   11: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=14.514, mean=0.25)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   12: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=17.503, mean=0.427)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   13: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=17.247, mean=0.239)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   14: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=16.555, mean=0.016)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   15: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=16.864, mean=0.017)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   16: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=6.092, mean=-0.508)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   17: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=17.553, mean=0.969)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   18: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=15.832, mean=-0.221)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   19: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=30.285, mean=-0.458)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   20: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=12.957, mean=-0.086)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   21: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=26.091, mean=-0.39)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   22: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=14.433, mean=-0.104)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   23: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=11.563, mean=-0.535)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   24: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=15.642, mean=-0.082)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   25: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=10.864, mean=-0.425)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   26: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=11.947, mean=0.192)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   27: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=13.466, mean=0.138)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   28: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=16.986, mean=-0.656)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   29: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=10.284, mean=0.065)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   30: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=16.697, mean=0.381)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   31: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=10.375, mean=-0.326)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   32: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=15.191, mean=0.233)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   33: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=20.617, mean=0.53)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   34: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=18.512, mean=0.525)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   35: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=9.414, mean=-0.446)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   36: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=11.822, mean=0.334)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   37: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=20.875, mean=0.224)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   38: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=23.327, mean=0.888)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   39: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=10.214, mean=-0.268)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   40: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=13.178, mean=-0.127)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   41: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=12.537, mean=-0.388)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   42: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=13.01, mean=-0.274)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   43: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=15.221, mean=0.13)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   44: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=15.895, mean=-0.301)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   45: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=21.116, mean=-0.148)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   46: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=18.923, mean=-0.063)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   47: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=15.774, mean=-0.557)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   48: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=16.196, mean=0.574)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   49: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=17.276, mean=0.222)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   50: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=10.08, mean=-0.646)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   51: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=14.076, mean=0.27)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   52: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=12.056, mean=-0.504)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   53: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=11.149, mean=-0.133)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   54: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=16.62, mean=0.564)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   55: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=9.953, mean=-0.382)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   56: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=20.508, mean=0.106)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   57: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=15.243, mean=0.04)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   58: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=14.563, mean=0.216)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   59: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=10.52, mean=0.115)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   60: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=9.542, mean=0.12)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   61: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=12.736, mean=-0.524)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   62: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=17.577, mean=0.376)}},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   63: { 'agent0': { 'action_mask': np.ndarray((284,), dtype=int64, min=0.0, max=1.0, mean=0.137),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                     'obs': np.ndarray((256,), dtype=float64, min=-1.0, max=13.834, mean=0.505)}}}\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m 2022-07-14 17:35:14,669\tINFO sampler.py:594 -- Info return from env: { 0: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   1: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   2: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   3: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   4: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   5: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   6: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   7: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   8: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   9: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   10: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   11: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   12: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   13: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   14: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   15: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   16: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   17: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   18: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   19: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   20: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   21: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   22: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   23: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   24: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   25: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   26: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   27: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   28: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   29: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   30: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   31: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   32: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   33: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   34: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   35: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   36: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   37: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   38: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   39: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   40: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   41: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   42: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   43: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   44: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   45: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   46: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   47: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   48: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   49: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   50: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   51: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   52: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   53: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   54: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   55: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   56: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   57: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   58: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   59: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   60: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   61: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   62: {'agent0': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   63: {'agent0': None}}\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m 2022-07-14 17:35:14,669\tINFO sampler.py:823 -- Preprocessed obs: np.ndarray((540,), dtype=float32, min=-1.0, max=14.742, mean=0.169)\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m 2022-07-14 17:35:14,669\tINFO sampler.py:827 -- Filtered obs: np.ndarray((540,), dtype=float32, min=-1.0, max=14.742, mean=0.169)\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m 2022-07-14 17:35:14,693\tINFO sampler.py:1017 -- Inputs to compute_actions():\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m \n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m { 'default_policy': [ { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 0,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=14.742, mean=0.169),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 1,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=6.099, mean=-0.113),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 2,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=10.2, mean=0.172),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 3,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=14.113, mean=0.113),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 4,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=15.212, mean=0.24),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 5,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=13.614, mean=0.056),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 6,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=13.649, mean=0.061),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 7,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=16.506, mean=0.265),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 8,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=14.946, mean=0.135),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 9,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=12.712, mean=0.06),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 10,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=10.496, mean=0.135),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 11,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=14.514, mean=0.191),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 12,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=17.503, mean=0.275),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 13,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=17.247, mean=0.185),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 14,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=16.555, mean=0.08),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 15,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=16.864, mean=0.08),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 16,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=6.092, mean=-0.169),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 17,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=17.553, mean=0.532),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 18,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=15.832, mean=-0.032),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 19,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=30.285, mean=-0.145),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 20,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=12.957, mean=0.031),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 21,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=26.091, mean=-0.113),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 22,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=14.433, mean=0.023),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 23,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=11.563, mean=-0.182),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 24,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=15.642, mean=0.033),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 25,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=10.864, mean=-0.129),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 26,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=11.947, mean=0.163),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 27,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=13.466, mean=0.138),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 28,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=16.986, mean=-0.239),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 29,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=10.284, mean=0.103),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 30,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=16.697, mean=0.253),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 31,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=10.375, mean=-0.082),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 32,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=15.191, mean=0.183),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 33,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=20.617, mean=0.324),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 34,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=18.512, mean=0.321),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 35,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=9.414, mean=-0.139),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 36,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=11.822, mean=0.231),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 37,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=20.875, mean=0.179),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 38,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=23.327, mean=0.493),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 39,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=10.214, mean=-0.055),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 40,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=13.178, mean=0.012),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 41,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=12.537, mean=-0.111),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 42,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=13.01, mean=-0.058),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 43,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=15.221, mean=0.134),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 44,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=15.895, mean=-0.071),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 45,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=21.116, mean=0.002),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 46,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=18.923, mean=0.042),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 47,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=15.774, mean=-0.192),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 48,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=16.196, mean=0.345),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 49,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=17.276, mean=0.177),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 50,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=10.08, mean=-0.234),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 51,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=14.076, mean=0.2),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 52,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=12.056, mean=-0.167),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 53,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=11.149, mean=0.009),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 54,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=16.62, mean=0.34),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 55,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=9.953, mean=-0.109),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 56,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=20.508, mean=0.123),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 57,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=15.243, mean=0.091),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 58,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=14.563, mean=0.174),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 59,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=10.52, mean=0.127),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 60,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=9.542, mean=0.129),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 61,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=12.736, mean=-0.176),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 62,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=17.577, mean=0.25),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'data': { 'agent_id': 'agent0',\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'env_id': 63,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'info': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'obs': np.ndarray((540,), dtype=float32, min=-1.0, max=13.834, mean=0.312),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_action': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'prev_reward': None,\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                                   'rnn_state': None},\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'type': 'PolicyEvalData'}]}\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m \n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m 2022-07-14 17:35:14,751\tINFO sampler.py:1043 -- Outputs of compute_actions():\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m \n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m { 'default_policy': ( np.ndarray((64,), dtype=int64, min=8.0, max=273.0, mean=150.266),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       [],\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                       { 'action_dist_inputs': np.ndarray((64, 284), dtype=float32, min=-0.023, max=0.025, mean=0.0),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'action_logp': np.ndarray((64,), dtype=float32, min=-5.658, max=-5.633, mean=-5.648),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'action_prob': np.ndarray((64,), dtype=float32, min=0.003, max=0.004, mean=0.004),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m                         'vf_preds': np.ndarray((64,), dtype=float32, min=-0.012, max=0.007, mean=-0.001)})}\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m 2022-07-14 17:35:17,243\tINFO simple_list_collector.py:661 -- Trajectory fragment after postprocess_trajectory():\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m \n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m { 'agent0': { 'action_dist_inputs': np.ndarray((9, 284), dtype=float32, min=-0.017, max=0.021, mean=0.0),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m               'action_logp': np.ndarray((9,), dtype=float32, min=-5.656, max=-5.641, mean=-5.648),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m               'actions': np.ndarray((9,), dtype=int64, min=51.0, max=263.0, mean=165.667),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m               'advantages': np.ndarray((9,), dtype=float32, min=-0.002, max=0.006, mean=0.001),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m               'agent_index': np.ndarray((9,), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m               'dones': np.ndarray((9,), dtype=bool, min=0.0, max=1.0, mean=0.111),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m               'eps_id': np.ndarray((9,), dtype=int64, min=219111070.0, max=219111070.0, mean=219111070.0),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m               'infos': np.ndarray((9,), dtype=object, head={}),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m               'new_obs': np.ndarray((9, 540), dtype=float32, min=-1.0, max=29.173, mean=0.693),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m               'obs': np.ndarray((9, 540), dtype=float32, min=-1.0, max=29.173, mean=0.653),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m               'rewards': np.ndarray((9,), dtype=float32, min=0.0, max=0.0, mean=0.0),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m               'unroll_id': np.ndarray((9,), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m               'value_targets': np.ndarray((9,), dtype=float32, min=0.0, max=0.0, mean=0.0),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m               'vf_preds': np.ndarray((9,), dtype=float32, min=-0.006, max=0.002, mean=-0.001)}}\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m \n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m 2022-07-14 17:35:17,523\tINFO rollout_worker.py:775 -- Completed sample batch:\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m \n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m { 'action_dist_inputs': np.ndarray((9, 284), dtype=float32, min=-0.017, max=0.021, mean=0.0),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   'action_logp': np.ndarray((9,), dtype=float32, min=-5.656, max=-5.641, mean=-5.648),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   'actions': np.ndarray((9,), dtype=int64, min=51.0, max=263.0, mean=165.667),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   'advantages': np.ndarray((9,), dtype=float32, min=-0.002, max=0.006, mean=0.001),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   'agent_index': np.ndarray((9,), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   'dones': np.ndarray((9,), dtype=bool, min=0.0, max=1.0, mean=0.111),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   'eps_id': np.ndarray((9,), dtype=int64, min=219111070.0, max=219111070.0, mean=219111070.0),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   'obs': np.ndarray((9, 540), dtype=float32, min=-1.0, max=29.173, mean=0.653),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   'rewards': np.ndarray((9,), dtype=float32, min=0.0, max=0.0, mean=0.0),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   'unroll_id': np.ndarray((9,), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   'value_targets': np.ndarray((9,), dtype=float32, min=0.0, max=0.0, mean=0.0),\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m   'vf_preds': np.ndarray((9,), dtype=float32, min=-0.006, max=0.002, mean=-0.001)}\n",
      "\u001b[2m\u001b[36m(pid=47832)\u001b[0m \n",
      "2022-07-14 17:35:17,660\tINFO tf_policy.py:712 -- Optimizing variable <tf.Variable 'default_policy/fc_1/kernel:0' shape=(540, 256) dtype=float32>\n",
      "2022-07-14 17:35:17,661\tINFO tf_policy.py:712 -- Optimizing variable <tf.Variable 'default_policy/fc_1/bias:0' shape=(256,) dtype=float32>\n",
      "2022-07-14 17:35:17,662\tINFO tf_policy.py:712 -- Optimizing variable <tf.Variable 'default_policy/fc_value_1/kernel:0' shape=(540, 256) dtype=float32>\n",
      "2022-07-14 17:35:17,663\tINFO tf_policy.py:712 -- Optimizing variable <tf.Variable 'default_policy/fc_value_1/bias:0' shape=(256,) dtype=float32>\n",
      "2022-07-14 17:35:17,663\tINFO tf_policy.py:712 -- Optimizing variable <tf.Variable 'default_policy/fc_2/kernel:0' shape=(256, 256) dtype=float32>\n",
      "2022-07-14 17:35:17,664\tINFO tf_policy.py:712 -- Optimizing variable <tf.Variable 'default_policy/fc_2/bias:0' shape=(256,) dtype=float32>\n",
      "2022-07-14 17:35:17,665\tINFO tf_policy.py:712 -- Optimizing variable <tf.Variable 'default_policy/fc_value_2/kernel:0' shape=(256, 256) dtype=float32>\n",
      "2022-07-14 17:35:17,666\tINFO tf_policy.py:712 -- Optimizing variable <tf.Variable 'default_policy/fc_value_2/bias:0' shape=(256,) dtype=float32>\n",
      "2022-07-14 17:35:17,666\tINFO tf_policy.py:712 -- Optimizing variable <tf.Variable 'default_policy/fc_out/kernel:0' shape=(256, 284) dtype=float32>\n",
      "2022-07-14 17:35:17,667\tINFO tf_policy.py:712 -- Optimizing variable <tf.Variable 'default_policy/fc_out/bias:0' shape=(284,) dtype=float32>\n",
      "2022-07-14 17:35:17,667\tINFO tf_policy.py:712 -- Optimizing variable <tf.Variable 'default_policy/value_out/kernel:0' shape=(256, 1) dtype=float32>\n",
      "2022-07-14 17:35:17,668\tINFO tf_policy.py:712 -- Optimizing variable <tf.Variable 'default_policy/value_out/bias:0' shape=(1,) dtype=float32>\n",
      "2022-07-14 17:35:17,676\tINFO multi_gpu_impl.py:143 -- Training on concatenated sample batches:\n",
      "\n",
      "{ 'inputs': [ np.ndarray((576, 540), dtype=float32, min=-1.0, max=86.79, mean=0.546),\n",
      "              np.ndarray((576,), dtype=int64, min=1.0, max=283.0, mean=146.33),\n",
      "              np.ndarray((576,), dtype=float32, min=0.0, max=0.0, mean=0.0),\n",
      "              np.ndarray((576,), dtype=bool, min=0.0, max=1.0, mean=0.111),\n",
      "              np.ndarray((576,), dtype=float32, min=-5.662, max=-5.631, mean=-5.649),\n",
      "              np.ndarray((576, 284), dtype=float32, min=-0.024, max=0.027, mean=0.0),\n",
      "              np.ndarray((576,), dtype=float32, min=-0.02, max=0.012, mean=-0.0),\n",
      "              np.ndarray((576,), dtype=float32, min=-2.416, max=3.866, mean=0.0),\n",
      "              np.ndarray((576,), dtype=float32, min=0.0, max=0.0, mean=0.0)],\n",
      "  'placeholders': [ <tf.Tensor 'default_policy/obs:0' shape=(?, 540) dtype=float32>,\n",
      "                    <tf.Tensor 'default_policy/action:0' shape=(?,) dtype=int64>,\n",
      "                    <tf.Tensor 'default_policy/rewards:0' shape=(?,) dtype=float32>,\n",
      "                    <tf.Tensor 'default_policy/dones:0' shape=(?,) dtype=float32>,\n",
      "                    <tf.Tensor 'default_policy/action_logp:0' shape=(?,) dtype=float32>,\n",
      "                    <tf.Tensor 'default_policy/action_dist_inputs:0' shape=(?, 284) dtype=float32>,\n",
      "                    <tf.Tensor 'default_policy/vf_preds:0' shape=(?,) dtype=float32>,\n",
      "                    <tf.Tensor 'default_policy/advantages:0' shape=(?,) dtype=float32>,\n",
      "                    <tf.Tensor 'default_policy/value_targets:0' shape=(?,) dtype=float32>],\n",
      "  'state_inputs': []}\n",
      "\n",
      "2022-07-14 17:35:17,678\tINFO multi_gpu_impl.py:188 -- Divided 576 rollout sequences, each of length 1, among 1 devices.\n",
      "2022-07-14 17:35:18,225\tINFO trainer.py:931 -- Synchronizing weights to workers.\n",
      "2022-07-14 17:35:18,469\tINFO trainer.py:866 -- Evaluating current policy for 8192 episodes.\n",
      "2022-07-14 17:35:18,471\tINFO rollout_worker.py:737 -- Generating sample batch of size 9\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import gym\n",
    "import ray\n",
    "from ray.rllib.models import ModelCatalog\n",
    "from ray.tune.registry import register_env\n",
    "from rl4rs.utils.rllib_print import pretty_print\n",
    "from rl4rs.nets.rllib.rllib_rawstate_model import getTFModelWithRawState\n",
    "from rl4rs.nets.rllib.rllib_mask_model import getMaskActionsModel, \\\n",
    "    getMaskActionsModelWithRawState\n",
    "from rl4rs.utils.rllib_vector_env import MyVectorEnvWrapper\n",
    "from script.modelfree_trainer import get_rl_model\n",
    "from rl4rs.policy.behavior_model import behavior_model\n",
    "from script.offline_evaluation import ope_eval\n",
    "from rl4rs.utils.fileutil import find_newest_files\n",
    "import http.client\n",
    "import sys\n",
    "http.client.HTTPConnection._http_vsn = 10\n",
    "http.client.HTTPConnection._http_vsn_str = 'HTTP/1.0'\n",
    "\n",
    "algo = 'PPO'\n",
    "\n",
    "ray.init()\n",
    "\n",
    "config = {\"epoch\": 2, \"maxlen\": 64, \"batch_size\": 64, \"action_size\": 284, \n",
    "          \"class_num\": 2, \"dense_feature_num\": 432, \"category_feature_num\": 21, \n",
    "          \"category_hash_size\": 100000, \"seq_num\": 2, \"emb_size\": 128, \"is_eval\": False,\n",
    "          \"hidden_units\": 128, \"max_steps\": 9, \"action_emb_size\": 32,\n",
    "          \"sample_file\": '/project/wangkai/rl4rs_benchmark_materials/simulator/rl4rs_dataset_a_shuf.csv', \n",
    "          \"model_file\": \"/project/wangkai/rl4rs_benchmark_materials/simulator/finetuned/simulator_a_dien/model\",\n",
    "          \"iteminfo_file\": '/project/wangkai/rl4rs_benchmark_materials/raw_data/item_info.csv', \n",
    "          'remote_base': 'http://127.0.0.1:5000', 'trial_name': 'all',\n",
    "          \"support_rllib_mask\": True, 'env': \"SlateRecEnv-v0\"}\n",
    "\n",
    "print(config)\n",
    "\n",
    "mask_model = getMaskActionsModel(true_obs_shape=(256,), action_size=config['action_size'])\n",
    "ModelCatalog.register_custom_model(\"mask_model\", mask_model)\n",
    "mask_model_rawstate = getMaskActionsModelWithRawState(config=config, action_size=config['action_size'])\n",
    "ModelCatalog.register_custom_model(\"mask_model_rawstate\", mask_model_rawstate)\n",
    "model_rawstate = getTFModelWithRawState(config=config)\n",
    "ModelCatalog.register_custom_model(\"model_rawstate\", model_rawstate)\n",
    "register_env('rllibEnv-v0', lambda _: MyVectorEnvWrapper(gym.make('HttpEnv-v0', env_id=config['env'], config=config), config['batch_size']))\n",
    "\n",
    "cfg = {\n",
    "    \"num_workers\": 2,\n",
    "    \"use_critic\": True,\n",
    "    \"use_gae\": True,\n",
    "    \"lambda\": 1.0,\n",
    "    \"kl_coeff\": 0.2,\n",
    "    \"sgd_minibatch_size\": 256,\n",
    "    \"shuffle_sequences\": True,\n",
    "    \"num_sgd_iter\": 1,\n",
    "    \"lr\": 0.0001,\n",
    "    \"vf_loss_coeff\": 0.5,\n",
    "    \"clip_param\": 0.3,\n",
    "    \"vf_clip_param\": 500.0,\n",
    "    \"kl_target\": 0.01,\n",
    "}\n",
    "\n",
    "rllib_config = dict(\n",
    "    {\n",
    "        \"env\": \"rllibEnv-v0\",\n",
    "        \"gamma\": 1,\n",
    "        \"explore\": True,\n",
    "        \"exploration_config\": {\n",
    "            \"type\": \"SoftQ\",\n",
    "            # \"temperature\": 1.0,\n",
    "        },\n",
    "        \"num_gpus\": 1 if config.get('gpu', True) else 0,\n",
    "        \"num_workers\": 0,\n",
    "        \"framework\": 'tf',\n",
    "        \"rollout_fragment_length\": config['max_steps'],\n",
    "        \"batch_mode\": \"complete_episodes\",\n",
    "        \"train_batch_size\": min(config[\"batch_size\"] * config['max_steps'], 1024),\n",
    "        \"evaluation_interval\": 1,\n",
    "        \"evaluation_num_episodes\": 2048 * 4,\n",
    "        \"evaluation_config\": {\n",
    "            \"explore\": False\n",
    "        },\n",
    "        \"log_level\": \"INFO\",\n",
    "    },\n",
    "    **cfg)\n",
    "print('rllib_config', rllib_config)\n",
    "trainer = get_rl_model(algo, rllib_config)\n",
    "\n",
    "# restore_file = ''\n",
    "# trainer.restore(restore_file)\n",
    "\n",
    "for i in range(config[\"epoch\"]):\n",
    "    result = trainer.train()\n",
    "    if (i + 1) % 1 == 0 or i == 0:\n",
    "        print(pretty_print(result))\n",
    "\n",
    "\n",
    "ray.shutdown()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}