Skip to content

Commit

Permalink
Merge pull request #156 from zuoxingdong/add_ddpg2
Browse files Browse the repository at this point in the history 
update CEM
  • Loading branch information
@zuoxingdong
zuoxingdong committed May 6, 2019
2 parents 8da7574 + b439236 commit caf4c44
Show file tree
Hide file tree
Showing 11 changed files with 20 additions and 18 deletions.
30 changes: 16 additions & 14 deletions baselines/cem/experiment.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,14 @@
from lagom.envs.wrappers import VecStandardizeObservation

from lagom import CEM
from agent import Agent
from .agent import Agent


config = Config(
{'cuda': False,
'log.dir': 'logs/default',
'log.freq': 10,
'checkpoint.freq': 50,
'checkpoint.num': 3,

'env.id': Grid(['HalfCheetah-v3', 'Hopper-v3', 'Walker2d-v3', 'Swimmer-v3']),
'env.standardize_obs': False,
Expand Down Expand Up @@ -72,7 +72,7 @@ def initializer(config, seed, device):
env = make_env(config, seed)
env = VecMonitor(env)
if config['env.standardize_obs']:
env = VecStandardizeObservation(env, clip=10.)
env = VecStandardizeObservation(env, clip=5.)
global agent
agent = Agent(config, env, device)

Expand Down Expand Up @@ -100,32 +100,34 @@ def run(config, seed, device):

print('Initializing...')
agent = Agent(config, make_env(config, seed), device)
cem = CEM([config['train.mu0']]*agent.num_params, config['train.std0'],
{'popsize': config['train.popsize'],
'seed': seed,
'elite_ratio': config['train.elite_ratio'],
'noise_scheduler_args': config['train.noise_scheduler_args']})
es = CEM([config['train.mu0']]*agent.num_params, config['train.std0'],
{'popsize': config['train.popsize'],
'seed': seed,
'elite_ratio': config['train.elite_ratio'],
'noise_scheduler_args': config['train.noise_scheduler_args']})
train_logs = []
checkpoint_count = 0
with ProcessPoolExecutor(max_workers=config['train.popsize'], initializer=initializer, initargs=(config, seed, device)) as executor:
print('Finish initialization. Training starts...')
for generation in range(config['train.generations']):
start_time = time.perf_counter()
solutions = cem.ask()
solutions = es.ask()
out = list(executor.map(fitness, solutions, chunksize=4))
Rs, Hs = zip(*out)
cem.tell(solutions, [-R for R in Rs])
es.tell(solutions, [-R for R in Rs])
logger = Logger()
logger('generation', generation+1)
logger('num_seconds', round(time.perf_counter() - start_time, 1))
logger('Returns', describe(Rs, axis=-1, repr_indent=1, repr_prefix='\n'))
logger('Horizons', describe(Hs, axis=-1, repr_indent=1, repr_prefix='\n'))
logger('fbest', cem.result.fbest)
logger('fbest', es.result.fbest)
train_logs.append(logger.logs)
if generation == 0 or (generation+1)%config['log.freq'] == 0:
logger.dump(keys=None, index=0, indent=0, border='-'*50)
if generation == 0 or (generation+1)%config['checkpoint.freq'] == 0:
agent.from_vec(torch.from_numpy(cem.result.xbest).float())
if (generation+1) >= int(config['train.generations']*(checkpoint_count/(config['checkpoint.num'] - 1))):
agent.from_vec(torch.from_numpy(es.result.xbest).float())
agent.checkpoint(logdir, generation+1)
checkpoint_count += 1
pickle_dump(obj=train_logs, f=logdir/'train_logs', ext='.pkl')
return None

Expand All @@ -134,4 +136,4 @@ def run(config, seed, device):
run_experiment(run=run,
config=config,
seeds=[1770966829, 1500925526, 2054191100],
num_worker=1)
num_worker=5)
1 change: 1 addition & 0 deletions baselines/cem/logs/default/3/1500925526/anim/openaigym.episode_batch.0.1844300.stats.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"initial_reset_timestamp": 1557148834.2342746, "timestamps": [1557148843.36324], "episode_lengths": [1000], "episode_rewards": [357.36394325127907], "episode_types": ["t", "t"]}
1 change: 0 additions & 1 deletion baselines/cem/logs/default/3/1500925526/anim/openaigym.episode_batch.12.1635725.stats.json
View file

This file was deleted.

1 change: 1 addition & 0 deletions baselines/cem/logs/default/3/1500925526/anim/openaigym.manifest.0.1844300.manifest.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"stats": "openaigym.episode_batch.0.1844300.stats.json", "videos": [["openaigym.video.0.1844300.video000000.mp4", "openaigym.video.0.1844300.video000000.meta.json"], ["openaigym.video.0.1844300.video000001.mp4", "openaigym.video.0.1844300.video000001.meta.json"]], "env_info": {"gym_version": "0.12.1", "env_id": "Swimmer-v3"}}
1 change: 0 additions & 1 deletion baselines/cem/logs/default/3/1500925526/anim/openaigym.manifest.12.1635725.manifest.json
View file

This file was deleted.

2 changes: 1 addition & 1 deletion ...ym.video.12.1635725.video000000.meta.json → ...gym.video.0.1844300.video000000.meta.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"episode_id": 0, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil 55. 78.100 / 55. 78.100\\nlibavcodec 57.107.100 / 57.107.100\\nlibavformat 57. 83.100 / 57. 83.100\\nlibavdevice 57. 10.100 / 57. 10.100\\nlibavfilter 6.107.100 / 6.107.100\\nlibavresample 3. 7. 0 / 3. 7. 0\\nlibswscale 4. 8.100 / 4. 8.100\\nlibswresample 2. 9.100 / 2. 9.100\\nlibpostproc 54. 7.100 / 54. 7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "25", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/tmp/lagom/examples/reinforcement_learning/cem/logs/default/3/1500925526/anim/openaigym.video.12.1635725.video000000.mp4"]}}
{"episode_id": 0, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil 55. 78.100 / 55. 78.100\\nlibavcodec 57.107.100 / 57.107.100\\nlibavformat 57. 83.100 / 57. 83.100\\nlibavdevice 57. 10.100 / 57. 10.100\\nlibavfilter 6.107.100 / 6.107.100\\nlibavresample 3. 7. 0 / 3. 7. 0\\nlibswscale 4. 8.100 / 4. 8.100\\nlibswresample 2. 9.100 / 2. 9.100\\nlibpostproc 54. 7.100 / 54. 7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "25", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/tmp/lagom/baselines/cem/logs/default/3/1500925526/anim/openaigym.video.0.1844300.video000000.mp4"]}}
Binary file added baselines/cem/logs/default/3/1500925526/anim/openaigym.video.0.1844300.video000000.mp4
View file
Binary file not shown.
2 changes: 1 addition & 1 deletion ...ym.video.12.1635725.video000001.meta.json → ...gym.video.0.1844300.video000001.meta.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"episode_id": 1, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil 55. 78.100 / 55. 78.100\\nlibavcodec 57.107.100 / 57.107.100\\nlibavformat 57. 83.100 / 57. 83.100\\nlibavdevice 57. 10.100 / 57. 10.100\\nlibavfilter 6.107.100 / 6.107.100\\nlibavresample 3. 7. 0 / 3. 7. 0\\nlibswscale 4. 8.100 / 4. 8.100\\nlibswresample 2. 9.100 / 2. 9.100\\nlibpostproc 54. 7.100 / 54. 7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "25", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/tmp/lagom/examples/reinforcement_learning/cem/logs/default/3/1500925526/anim/openaigym.video.12.1635725.video000001.mp4"]}}
{"episode_id": 1, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil 55. 78.100 / 55. 78.100\\nlibavcodec 57.107.100 / 57.107.100\\nlibavformat 57. 83.100 / 57. 83.100\\nlibavdevice 57. 10.100 / 57. 10.100\\nlibavfilter 6.107.100 / 6.107.100\\nlibavresample 3. 7. 0 / 3. 7. 0\\nlibswscale 4. 8.100 / 4. 8.100\\nlibswresample 2. 9.100 / 2. 9.100\\nlibpostproc 54. 7.100 / 54. 7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "25", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/tmp/lagom/baselines/cem/logs/default/3/1500925526/anim/openaigym.video.0.1844300.video000001.mp4"]}}
0 ...penaigym.video.12.1635725.video000001.mp4 → ...openaigym.video.0.1844300.video000001.mp4
View file
File renamed without changes.
Binary file removed baselines/cem/logs/default/3/1500925526/anim/openaigym.video.12.1635725.video000000.mp4
View file
Binary file not shown.
Binary file modified baselines/cem/logs/default/result.png
View file
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.

0 comments on commit caf4c44

Please sign in to comment.