Merge pull request #179 from zuoxingdong/step_info_trajectory

update logs to VPG/PPO

baselines/ppo/agent.py

@@ -66,7 +66,7 @@ def __init__(self, config, env, device, **kwargs):
             ortho_init(layer, nonlinearity='tanh', constant_bias=0.0)
 
         feature_dim = config['nn.sizes'][-1]
-        self.V_head = nn.Linear(feature_dim, 1).to(device)
+        self.V_head = nn.Linear(feature_dim, 1)
         ortho_init(self.V_head, weight_scale=1.0, constant_bias=0.0)
 
         self.to(self.device)

baselines/ppo/experiment.py

@@ -103,5 +103,5 @@ def run(config, seed, device, logdir):
                    log_dir='logs/default',
                    max_workers=os.cpu_count(), 
                    chunksize=1, 
-                   use_gpu=False,  # CPU a bit faster
+                   use_gpu=False,  # CPU a bit faster, note that performance differs between CPU/GPU
                    gpu_ids=None)

baselines/ppo/logs/default/0/1500925526/anim/openaigym.episode_batch.0.392667.stats.json

@@ -0,0 +1 @@
+{"initial_reset_timestamp": 1557411883.1675007, "timestamps": [1557411895.9823985], "episode_lengths": [1000], "episode_rewards": [4663.648824806512], "episode_types": ["t", "t"]}

baselines/ppo/logs/default/0/1500925526/anim/openaigym.manifest.0.392667.manifest.json

@@ -0,0 +1 @@
+{"stats": "openaigym.episode_batch.0.392667.stats.json", "videos": [["openaigym.video.0.392667.video000000.mp4", "openaigym.video.0.392667.video000000.meta.json"], ["openaigym.video.0.392667.video000001.mp4", "openaigym.video.0.392667.video000001.meta.json"]], "env_info": {"gym_version": "0.12.1", "env_id": "HalfCheetah-v3"}}

baselines/ppo/logs/default/0/1500925526/anim/openaigym.video.10.2837515.video000000.meta.json → baselines/ppo/logs/default/0/1500925526/anim/openaigym.video.0.392667.video000000.meta.json

@@ -1 +1 @@
-{"episode_id": 0, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil      55. 78.100 / 55. 78.100\\nlibavcodec     57.107.100 / 57.107.100\\nlibavformat    57. 83.100 / 57. 83.100\\nlibavdevice    57. 10.100 / 57. 10.100\\nlibavfilter     6.107.100 /  6.107.100\\nlibavresample   3.  7.  0 /  3.  7.  0\\nlibswscale      4.  8.100 /  4.  8.100\\nlibswresample   2.  9.100 /  2.  9.100\\nlibpostproc    54.  7.100 / 54.  7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "20", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/lagom/baselines/ppo/logs/default/0/1500925526/anim/openaigym.video.10.2837515.video000000.mp4"]}}
+{"episode_id": 0, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil      55. 78.100 / 55. 78.100\\nlibavcodec     57.107.100 / 57.107.100\\nlibavformat    57. 83.100 / 57. 83.100\\nlibavdevice    57. 10.100 / 57. 10.100\\nlibavfilter     6.107.100 /  6.107.100\\nlibavresample   3.  7.  0 /  3.  7.  0\\nlibswscale      4.  8.100 /  4.  8.100\\nlibswresample   2.  9.100 /  2.  9.100\\nlibpostproc    54.  7.100 / 54.  7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "20", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/lagom/baselines/ppo/logs/default/0/1500925526/anim/openaigym.video.0.392667.video000000.mp4"]}}

baselines/ppo/logs/default/0/1500925526/anim/openaigym.video.10.2837515.video000001.meta.json → baselines/ppo/logs/default/0/1500925526/anim/openaigym.video.0.392667.video000001.meta.json

@@ -1 +1 @@
-{"episode_id": 1, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil      55. 78.100 / 55. 78.100\\nlibavcodec     57.107.100 / 57.107.100\\nlibavformat    57. 83.100 / 57. 83.100\\nlibavdevice    57. 10.100 / 57. 10.100\\nlibavfilter     6.107.100 /  6.107.100\\nlibavresample   3.  7.  0 /  3.  7.  0\\nlibswscale      4.  8.100 /  4.  8.100\\nlibswresample   2.  9.100 /  2.  9.100\\nlibpostproc    54.  7.100 / 54.  7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "20", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/lagom/baselines/ppo/logs/default/0/1500925526/anim/openaigym.video.10.2837515.video000001.mp4"]}}
+{"episode_id": 1, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil      55. 78.100 / 55. 78.100\\nlibavcodec     57.107.100 / 57.107.100\\nlibavformat    57. 83.100 / 57. 83.100\\nlibavdevice    57. 10.100 / 57. 10.100\\nlibavfilter     6.107.100 /  6.107.100\\nlibavresample   3.  7.  0 /  3.  7.  0\\nlibswscale      4.  8.100 /  4.  8.100\\nlibswresample   2.  9.100 /  2.  9.100\\nlibpostproc    54.  7.100 / 54.  7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "20", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/lagom/baselines/ppo/logs/default/0/1500925526/anim/openaigym.video.0.392667.video000001.mp4"]}}

baselines/ppo/logs/default/source_files/agent.py

@@ -66,7 +66,7 @@ def __init__(self, config, env, device, **kwargs):
             ortho_init(layer, nonlinearity='tanh', constant_bias=0.0)
 
         feature_dim = config['nn.sizes'][-1]
-        self.V_head = nn.Linear(feature_dim, 1).to(device)
+        self.V_head = nn.Linear(feature_dim, 1)
         ortho_init(self.V_head, weight_scale=1.0, constant_bias=0.0)
 
         self.to(self.device)

baselines/ppo/logs/default/source_files/experiment.py

@@ -103,5 +103,5 @@ def run(config, seed, device, logdir):
                    log_dir='logs/default',
                    max_workers=os.cpu_count(), 
                    chunksize=1, 
-                   use_gpu=False,  # CPU a bit faster
+                   use_gpu=False,###True,######False,  # CPU a bit faster
                    gpu_ids=None)

baselines/vpg/agent.py

@@ -55,8 +55,9 @@ def __init__(self, config, env, device, **kwargs):
             self.action_head = CategoricalHead(feature_dim, env.action_space.n, device, **kwargs)
         elif isinstance(env.action_space, Box):
             self.action_head = DiagGaussianHead(feature_dim, flatdim(env.action_space), device, config['agent.std0'], **kwargs)
-        self.V_head = nn.Linear(feature_dim, 1).to(device)
+        self.V_head = nn.Linear(feature_dim, 1)
         ortho_init(self.V_head, weight_scale=1.0, constant_bias=0.0)
+        self.V_head = self.V_head.to(device)  # reproducible between CPU/GPU, ortho_init behaves differently
 
         self.total_timestep = 0
 

baselines/vpg/logs/default/source_files/agent.py

@@ -55,8 +55,9 @@ def __init__(self, config, env, device, **kwargs):
             self.action_head = CategoricalHead(feature_dim, env.action_space.n, device, **kwargs)
         elif isinstance(env.action_space, Box):
             self.action_head = DiagGaussianHead(feature_dim, flatdim(env.action_space), device, config['agent.std0'], **kwargs)
-        self.V_head = nn.Linear(feature_dim, 1).to(device)
+        self.V_head = nn.Linear(feature_dim, 1)
         ortho_init(self.V_head, weight_scale=1.0, constant_bias=0.0)
+        self.V_head = self.V_head.to(device)  # reproducible between CPU/GPU, ortho_init behaves differently
 
         self.total_timestep = 0