Merge pull request #190 from zuoxingdong/new_CI

update new loggings for CMA-ES

baselines/cmaes/logs/default/0/config.yml

@@ -1,14 +1,12 @@
 ID: 0
-cuda: false
-log.dir: logs/default
 log.freq: 10
-checkpoint.freq: 50
+checkpoint.num: 3
 env.standardize_obs: false
 nn.sizes:
 - 64
 - 64
 env.clip_action: true
-agent.std0: 0.5
+agent.std0: 0.6
 train.generations: 1000
 train.popsize: 64
 train.mu0: 0.0

baselines/cmaes/logs/default/1/config.yml

@@ -1,14 +1,12 @@
 ID: 1
-cuda: false
-log.dir: logs/default
 log.freq: 10
-checkpoint.freq: 50
+checkpoint.num: 3
 env.standardize_obs: false
 nn.sizes:
 - 64
 - 64
 env.clip_action: true
-agent.std0: 0.5
+agent.std0: 0.6
 train.generations: 1000
 train.popsize: 64
 train.mu0: 0.0

baselines/cmaes/logs/default/2/config.yml

@@ -1,14 +1,12 @@
 ID: 2
-cuda: false
-log.dir: logs/default
 log.freq: 10
-checkpoint.freq: 50
+checkpoint.num: 3
 env.standardize_obs: false
 nn.sizes:
 - 64
 - 64
 env.clip_action: true
-agent.std0: 0.5
+agent.std0: 0.6
 train.generations: 1000
 train.popsize: 64
 train.mu0: 0.0

baselines/cmaes/logs/default/3/1500925526/anim/openaigym.episode_batch.7.1637806.stats.json

@@ -0,0 +1 @@
+{"initial_reset_timestamp": 1557912770.1625261, "timestamps": [1557912799.5660088], "episode_lengths": [1000], "episode_rewards": [353.18737583470426], "episode_types": ["t", "t"]}

baselines/cmaes/logs/default/3/1500925526/anim/openaigym.manifest.7.1637806.manifest.json

@@ -0,0 +1 @@
+{"stats": "openaigym.episode_batch.7.1637806.stats.json", "videos": [["openaigym.video.7.1637806.video000000.mp4", "openaigym.video.7.1637806.video000000.meta.json"], ["openaigym.video.7.1637806.video000001.mp4", "openaigym.video.7.1637806.video000001.meta.json"]], "env_info": {"gym_version": "0.12.1", "env_id": "Swimmer-v3"}}

baselines/cmaes/logs/default/1/1500925526/anim/openaigym.video.0.1865711.video000000.meta.json → baselines/cmaes/logs/default/3/1500925526/anim/openaigym.video.7.1637806.video000000.meta.json

@@ -1 +1 @@
-{"episode_id": 0, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil      55. 78.100 / 55. 78.100\\nlibavcodec     57.107.100 / 57.107.100\\nlibavformat    57. 83.100 / 57. 83.100\\nlibavdevice    57. 10.100 / 57. 10.100\\nlibavfilter     6.107.100 /  6.107.100\\nlibavresample   3.  7.  0 /  3.  7.  0\\nlibswscale      4.  8.100 /  4.  8.100\\nlibswresample   2.  9.100 /  2.  9.100\\nlibpostproc    54.  7.100 / 54.  7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "125", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/tmp/lagom/baselines/cmaes/logs/default/1/1500925526/anim/openaigym.video.0.1865711.video000000.mp4"]}}
+{"episode_id": 0, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil      55. 78.100 / 55. 78.100\\nlibavcodec     57.107.100 / 57.107.100\\nlibavformat    57. 83.100 / 57. 83.100\\nlibavdevice    57. 10.100 / 57. 10.100\\nlibavfilter     6.107.100 /  6.107.100\\nlibavresample   3.  7.  0 /  3.  7.  0\\nlibswscale      4.  8.100 /  4.  8.100\\nlibswresample   2.  9.100 /  2.  9.100\\nlibpostproc    54.  7.100 / 54.  7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "25", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/lagom/baselines/cmaes/logs/default/3/1500925526/anim/openaigym.video.7.1637806.video000000.mp4"]}}

baselines/cmaes/logs/default/1/1500925526/anim/openaigym.video.0.1865711.video000001.meta.json → baselines/cmaes/logs/default/3/1500925526/anim/openaigym.video.7.1637806.video000001.meta.json

@@ -1 +1 @@
-{"episode_id": 1, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil      55. 78.100 / 55. 78.100\\nlibavcodec     57.107.100 / 57.107.100\\nlibavformat    57. 83.100 / 57. 83.100\\nlibavdevice    57. 10.100 / 57. 10.100\\nlibavfilter     6.107.100 /  6.107.100\\nlibavresample   3.  7.  0 /  3.  7.  0\\nlibswscale      4.  8.100 /  4.  8.100\\nlibswresample   2.  9.100 /  2.  9.100\\nlibpostproc    54.  7.100 / 54.  7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "125", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/tmp/lagom/baselines/cmaes/logs/default/1/1500925526/anim/openaigym.video.0.1865711.video000001.mp4"]}}
+{"episode_id": 1, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil      55. 78.100 / 55. 78.100\\nlibavcodec     57.107.100 / 57.107.100\\nlibavformat    57. 83.100 / 57. 83.100\\nlibavdevice    57. 10.100 / 57. 10.100\\nlibavfilter     6.107.100 /  6.107.100\\nlibavresample   3.  7.  0 /  3.  7.  0\\nlibswscale      4.  8.100 /  4.  8.100\\nlibswresample   2.  9.100 /  2.  9.100\\nlibpostproc    54.  7.100 / 54.  7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "25", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/lagom/baselines/cmaes/logs/default/3/1500925526/anim/openaigym.video.7.1637806.video000001.mp4"]}}

baselines/cmaes/logs/default/3/config.yml

@@ -1,14 +1,12 @@
 ID: 3
-cuda: false
-log.dir: logs/default
 log.freq: 10
-checkpoint.freq: 50
+checkpoint.num: 3
 env.standardize_obs: false
 nn.sizes:
 - 64
 - 64
 env.clip_action: true
-agent.std0: 0.5
+agent.std0: 0.6
 train.generations: 1000
 train.popsize: 64
 train.mu0: 0.0

baselines/cmaes/logs/default/source_files/agent.py

@@ -9,6 +9,8 @@
 
 from lagom import BaseAgent
 from lagom.utils import pickle_dump
+from lagom.utils import tensorify
+from lagom.utils import numpify
 from lagom.envs import flatdim
 from lagom.envs.wrappers import get_wrapper
 from lagom.networks import Module
@@ -47,15 +49,14 @@ def __init__(self, config, env, device, **kwargs):
         self.total_timestep = 0
 
     def choose_action(self, obs, **kwargs):
-        if not torch.is_tensor(obs):
-            obs = torch.from_numpy(np.asarray(obs)).float().to(self.device)
+        obs = tensorify(obs, self.device)
         out = {}
         features = self.feature_network(obs)
 
         action_dist = self.action_head(features)
         out['entropy'] = action_dist.entropy()
         action = action_dist.sample()
-        out['raw_action'] = action.detach().cpu().numpy()
+        out['raw_action'] = numpify(action, 'float')
         return out
 
     def learn(self, D, **kwargs):

baselines/cmaes/logs/default/source_files/experiment.py

@@ -14,6 +14,7 @@
 from lagom.transform import describe
 from lagom.utils import CloudpickleWrapper  # VERY IMPORTANT
 from lagom.utils import pickle_dump
+from lagom.utils import tensorify
 from lagom.utils import set_global_seeds
 from lagom.experiment import Config
 from lagom.experiment import Grid
@@ -26,14 +27,12 @@
 from lagom.envs.wrappers import VecStandardizeObservation
 
 from lagom import CMAES
-from agent import Agent
+from baselines.cmaes.agent import Agent
 
 
 config = Config(
-    {'cuda': False, 
-     'log.dir': 'logs/default', 
-     'log.freq': 10, 
-     'checkpoint.freq': 50,
+    {'log.freq': 10, 
+     'checkpoint.num': 3,
 
      'env.id': Grid(['HalfCheetah-v3', 'Hopper-v3', 'Walker2d-v3', 'Swimmer-v3']), 
      'env.standardize_obs': False,
@@ -42,7 +41,7 @@
 
      # only for continuous control
      'env.clip_action': True,  # clip action within valid bound before step()
-     'agent.std0': 0.5,  # initial std
+     'agent.std0': 0.6,  # initial std
 
      'train.generations': int(1e3),  # total number of ES generations
      'train.popsize': 64,
@@ -70,13 +69,13 @@ def initializer(config, seed, device):
     env = make_env(config, seed)
     env = VecMonitor(env)
     if config['env.standardize_obs']:
-        env = VecStandardizeObservation(env, clip=10.)
+        env = VecStandardizeObservation(env, clip=5.)
     global agent
     agent = Agent(config, env, device)
 
 
 def fitness(param):
-    agent.from_vec(torch.from_numpy(param).float())
+    agent.from_vec(tensorify(param, 'cpu'))
     R = []
     H = []
     with torch.no_grad():
@@ -92,22 +91,22 @@ def fitness(param):
     return np.mean(R), np.mean(H)
 
 
-def run(config, seed, device):
+def run(config, seed, device, logdir):
     set_global_seeds(seed)
-    logdir = Path(config['log.dir']) / str(config['ID']) / str(seed)
 
     print('Initializing...')
     agent = Agent(config, make_env(config, seed), device)
     es = CMAES([config['train.mu0']]*agent.num_params, config['train.std0'], 
                {'popsize': config['train.popsize'], 
                 'seed': seed})
     train_logs = []
+    checkpoint_count = 0
     with ProcessPoolExecutor(max_workers=config['train.popsize'], initializer=initializer, initargs=(config, seed, device)) as executor:
         print('Finish initialization. Training starts...')
         for generation in range(config['train.generations']):
             start_time = time.perf_counter()
             solutions = es.ask()
-            out = list(executor.map(fitness, solutions, chunksize=4))
+            out = list(executor.map(fitness, solutions, chunksize=2))
             Rs, Hs = zip(*out)
             es.tell(solutions, [-R for R in Rs])
             logger = Logger()
@@ -119,9 +118,10 @@ def run(config, seed, device):
             train_logs.append(logger.logs)
             if generation == 0 or (generation+1)%config['log.freq'] == 0:
                 logger.dump(keys=None, index=0, indent=0, border='-'*50)
-            if generation == 0 or (generation+1)%config['checkpoint.freq'] == 0:
-                agent.from_vec(torch.from_numpy(es.result.xbest).float())
+            if (generation+1) >= int(config['train.generations']*(checkpoint_count/(config['checkpoint.num'] - 1))):
+                agent.from_vec(tensorify(es.result.xbest, 'cpu'))
                 agent.checkpoint(logdir, generation+1)
+                checkpoint_count += 1
     pickle_dump(obj=train_logs, f=logdir/'train_logs', ext='.pkl')
     return None
 
@@ -130,4 +130,8 @@ def run(config, seed, device):
     run_experiment(run=run, 
                    config=config, 
                    seeds=[1770966829, 1500925526, 2054191100], 
-                   num_worker=1)
+                   log_dir='logs/default',
+                   max_workers=None,  # no parallelization 
+                   chunksize=1, 
+                   use_gpu=False,
+                   gpu_ids=None)