Merge pull request #168 from zuoxingdong/step_info_trajectory

Update VPG

baselines/vpg/experiment.py

@@ -17,14 +17,15 @@
 from lagom.envs.wrappers import VecMonitor
 from lagom.envs.wrappers import VecStandardizeObservation
 from lagom.envs.wrappers import VecStandardizeReward
+from lagom.envs.wrappers import VecStepInfo
 from lagom.runner import EpisodeRunner
 
 from baselines.vpg.agent import Agent
 from baselines.vpg.engine import Engine
 
 
 config = Config(
-    {'cuda': True, 
+    {'cuda': False,  # CPU a bit faster
      'log.dir': 'logs/default', 
      'log.freq': 10, 
      'checkpoint.num': 3,
@@ -76,6 +77,7 @@ def run(config, seed, device):
         env = VecStandardizeObservation(env, clip=5.)
     if config['env.standardize_reward']:
         env = VecStandardizeReward(env, clip=10., gamma=config['agent.gamma'])
+    env = VecStepInfo(env)
 
     agent = Agent(config, env, device)
     runner = EpisodeRunner(reset_on_call=False)

baselines/vpg/logs/default/1/1500925526/anim/openaigym.episode_batch.0.2837515.stats.json

@@ -0,0 +1 @@
+{"initial_reset_timestamp": 1557242203.1711683, "timestamps": [1557242222.817026], "episode_lengths": [1000], "episode_rewards": [3175.9709792188214], "episode_types": ["t", "t"]}

baselines/vpg/logs/default/1/1500925526/anim/openaigym.manifest.0.2837515.manifest.json

@@ -0,0 +1 @@
+{"stats": "openaigym.episode_batch.0.2837515.stats.json", "videos": [["openaigym.video.0.2837515.video000000.mp4", "openaigym.video.0.2837515.video000000.meta.json"], ["openaigym.video.0.2837515.video000001.mp4", "openaigym.video.0.2837515.video000001.meta.json"]], "env_info": {"gym_version": "0.12.1", "env_id": "Hopper-v3"}}

baselines/vpg/logs/default/1/1500925526/anim/openaigym.video.0.2837515.video000000.meta.json

@@ -0,0 +1 @@
+{"episode_id": 0, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil      55. 78.100 / 55. 78.100\\nlibavcodec     57.107.100 / 57.107.100\\nlibavformat    57. 83.100 / 57. 83.100\\nlibavdevice    57. 10.100 / 57. 10.100\\nlibavfilter     6.107.100 /  6.107.100\\nlibavresample   3.  7.  0 /  3.  7.  0\\nlibswscale      4.  8.100 /  4.  8.100\\nlibswresample   2.  9.100 /  2.  9.100\\nlibpostproc    54.  7.100 / 54.  7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "125", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/lagom/baselines/vpg/logs/default/1/1500925526/anim/openaigym.video.0.2837515.video000000.mp4"]}}

baselines/vpg/logs/default/1/1500925526/anim/openaigym.video.0.2837515.video000001.meta.json

@@ -0,0 +1 @@
+{"episode_id": 1, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil      55. 78.100 / 55. 78.100\\nlibavcodec     57.107.100 / 57.107.100\\nlibavformat    57. 83.100 / 57. 83.100\\nlibavdevice    57. 10.100 / 57. 10.100\\nlibavfilter     6.107.100 /  6.107.100\\nlibavresample   3.  7.  0 /  3.  7.  0\\nlibswscale      4.  8.100 /  4.  8.100\\nlibswresample   2.  9.100 /  2.  9.100\\nlibpostproc    54.  7.100 / 54.  7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "125", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/lagom/baselines/vpg/logs/default/1/1500925526/anim/openaigym.video.0.2837515.video000001.mp4"]}}

baselines/vpg/logs/default/source_files/experiment.py

@@ -17,14 +17,15 @@
 from lagom.envs.wrappers import VecMonitor
 from lagom.envs.wrappers import VecStandardizeObservation
 from lagom.envs.wrappers import VecStandardizeReward
+from lagom.envs.wrappers import VecStepInfo
 from lagom.runner import EpisodeRunner
 
 from baselines.vpg.agent import Agent
 from baselines.vpg.engine import Engine
 
 
 config = Config(
-    {'cuda': False,############True, 
+    {'cuda': False,  # CPU a bit faster
      'log.dir': 'logs/default', 
      'log.freq': 10, 
      'checkpoint.num': 3,
@@ -76,6 +77,7 @@ def run(config, seed, device):
         env = VecStandardizeObservation(env, clip=5.)
     if config['env.standardize_reward']:
         env = VecStandardizeReward(env, clip=10., gamma=config['agent.gamma'])
+    env = VecStepInfo(env)
 
     agent = Agent(config, env, device)
     runner = EpisodeRunner(reset_on_call=False)

lagom/runner/trajectory.py

@@ -52,6 +52,8 @@ def numpy_rewards(self):
     def add_step_info(self, step_info):
         assert not self.completed
         self.step_infos.append(step_info)
+        if step_info.last:
+            assert self.completed
 
     @property
     def numpy_dones(self):

requirements.txt

@@ -1,4 +1,4 @@
-# Update: 2019-04-24
+# Update: 2019-05-07
 
 # System-related
 mkl>=2019.0
@@ -39,9 +39,9 @@ matplotlib>=3.0.3
 seaborn>=0.9.0
 
 # RL
+atari_py>=0.1.7
 #gym>=0.12.1  # revert when gym update version with latest TimeLimit
 git+https://github.com/openai/gym.git
-atari_py>=0.1.7
 cma>=2.7.0
 
 # ReadtheDocs

scripts/3_setup_envs.sh

@@ -14,6 +14,16 @@ echo "set tabstop=4" >> ~/.vimrc
 echo "set shiftwidth=4" >> ~/.vimrc
 echo "set expandtab" >> ~/.vimrc
 
+# Install Mujoco
+wget https://www.roboti.us/download/mujoco200_linux.zip
+mkdir -p ~/.mujoco
+unzip mujoco200_linux.zip -d ~/.mujoco
+ln -s ~/.mujoco/mujoco200_linux ~/.mujoco/mujoco200
+echo "# Mujoco" >> ~/.bashrc
+echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:$HOME/.mujoco/mujoco200/bin:$HOME/.mujoco/mujoco200_linux/bin" >> ~/.bashrc
+echo ""  >> ~/.bashrc
+rm mujoco200_linux.zip
+
 # Install dependencies
 pip install --upgrade pip
 pip install -q -r ../requirements.txt