Merge pull request #161 from zuoxingdong/add_ddpg2

minor update

baselines/openaies/experiment.py

@@ -48,7 +48,7 @@
      'train.popsize': 64,
      'train.mu0': 0.0,
      'train.std0': 1.0,
-     'train.lr': 1e-1,
+     'train.lr': 1e-2,
      'train.lr_decay': 1.0,
      'train.min_lr': 1e-6,
      'train.sigma_scheduler_args': [1.0, 0.01, 450, 0],

baselines/ppo/agent.py

@@ -9,6 +9,8 @@
 
 from lagom import BaseAgent
 from lagom.utils import pickle_dump
+from lagom.utils import tensorify
+from lagom.utils import numpify
 from lagom.envs import flatdim
 from lagom.envs.wrappers import get_wrapper
 from lagom.networks import Module
@@ -23,7 +25,7 @@
 from lagom.transform import describe
 
 from torch.utils.data import DataLoader
-from dataset import Dataset
+from .dataset import Dataset
 
 
 class MLP(Module):
@@ -65,8 +67,7 @@ def __init__(self, config, env, device, **kwargs):
             self.lr_scheduler = linear_lr_scheduler(self.optimizer, config['train.timestep'], min_lr=1e-8)
 
     def choose_action(self, obs, **kwargs):
-        if not torch.is_tensor(obs):
-            obs = torch.from_numpy(np.asarray(obs)).float().to(self.device)
+        obs = tensorify(obs, self.device)
         out = {}
         features = self.feature_network(obs)
 
@@ -76,7 +77,7 @@ def choose_action(self, obs, **kwargs):
 
         action = action_dist.sample()
         out['action'] = action
-        out['raw_action'] = action.detach().cpu().numpy()
+        out['raw_action'] = numpify(action, 'float')
         out['action_logprob'] = action_dist.log_prob(action.detach())
 
         V = self.V_head(features)
@@ -117,7 +118,7 @@ def learn_one_update(self, data):
         out['entropy_loss'] = entropy_loss.mean().item()
         out['policy_entropy'] = -entropy_loss.mean().item()
         out['value_loss'] = value_loss.mean().item()
-        out['explained_variance'] = ev(y_true=old_Qs.detach().cpu().numpy(), y_pred=Vs.detach().cpu().numpy())
+        out['explained_variance'] = ev(y_true=numpify(old_Qs, 'float'), y_pred=numpify(Vs, 'float'))
         out['approx_kl'] = torch.mean(old_logprobs - logprobs).item()
         out['clip_frac'] = ((ratio < 1.0 - eps) | (ratio > 1.0 + eps)).float().mean().item()
         return out
@@ -128,17 +129,17 @@ def learn(self, D, **kwargs):
         entropies = [torch.cat(traj.get_all_info('entropy')) for traj in D]
         Vs = [torch.cat(traj.get_all_info('V')) for traj in D]
 
-        last_observations = torch.from_numpy(np.concatenate([traj.last_observation for traj in D], 0)).float()
         with torch.no_grad():
-            last_Vs = self.V_head(self.feature_network(last_observations.to(self.device))).squeeze(-1)
+            last_observations = tensorify(np.concatenate([traj.last_observation for traj in D], 0), self.device)
+            last_Vs = self.V_head(self.feature_network(last_observations)).squeeze(-1)
         Qs = [bootstrapped_returns(self.config['agent.gamma'], traj, last_V) 
                   for traj, last_V in zip(D, last_Vs)]
         As = [gae(self.config['agent.gamma'], self.config['agent.gae_lambda'], traj, V, last_V) 
                   for traj, V, last_V in zip(D, Vs, last_Vs)]
 
         # Metrics -> Tensor, device
         logprobs, entropies, Vs = map(lambda x: torch.cat(x).squeeze(), [logprobs, entropies, Vs])
-        Qs, As = map(lambda x: torch.from_numpy(np.concatenate(x).copy()).to(self.device), [Qs, As])
+        Qs, As = map(lambda x: tensorify(np.concatenate(x).copy(), self.device), [Qs, As])
         if self.config['agent.standardize_adv']:
             As = (As - As.mean())/(As.std() + 1e-8)
 

baselines/ppo/dataset.py

@@ -1,18 +1,18 @@
 import numpy as np
 
 from torch.utils import data
+from lagom.utils import numpify
 
 
 class Dataset(data.Dataset):
     def __init__(self, D, logprobs, entropies, Vs, Qs, As):
         self.observations = np.concatenate([np.concatenate(traj.observations[:-1], 0) for traj in D], 0).astype(np.float32)
-        self.actions = np.concatenate([traj.numpy_actions for traj in D], 0)
-        tensor_to_numpy = lambda x: x.detach().cpu().numpy()
-        self.logprobs = tensor_to_numpy(logprobs)
-        self.entropies = tensor_to_numpy(entropies)
-        self.Vs = tensor_to_numpy(Vs)
-        self.Qs = tensor_to_numpy(Qs)
-        self.As = tensor_to_numpy(As)
+        self.actions = np.concatenate([traj.numpy_actions for traj in D], 0).astype(np.float32)
+        self.logprobs = numpify(logprobs, 'float32')
+        self.entropies = numpify(entropies, 'float32')
+        self.Vs = numpify(Vs, 'float32')
+        self.Qs = numpify(Qs, 'float32')
+        self.As = numpify(As, 'float32')
 
         assert self.actions.shape[0] == len(self)
         assert all([item.shape == (len(self),) for item in [self.logprobs, self.entropies, 

baselines/ppo/experiment.py

@@ -19,15 +19,15 @@
 from lagom.envs.wrappers import VecStandardizeReward
 from lagom.runner import EpisodeRunner
 
-from agent import Agent
-from engine import Engine
+from .agent import Agent
+from .engine import Engine
 
 
 config = Config(
     {'cuda': True, 
      'log.dir': 'logs/default', 
      'log.freq': 10, 
-     'checkpoint.freq': 50,
+     'checkpoint.num': 3,
 
      'env.id': Grid(['HalfCheetah-v3', 'Hopper-v3', 'Walker2d-v3', 'Swimmer-v3']), 
      'env.standardize_obs': True,
@@ -83,16 +83,17 @@ def run(config, seed, device):
     runner = EpisodeRunner(reset_on_call=False)
     engine = Engine(config, agent=agent, env=env, runner=runner)
     train_logs = []
+    checkpoint_count = 0
     for i in count():
         if agent.total_timestep >= config['train.timestep']:
             break
         train_logger = engine.train(i)
         train_logs.append(train_logger.logs)
         if i == 0 or (i+1) % config['log.freq'] == 0:
             train_logger.dump(keys=None, index=0, indent=0, border='-'*50)
-        if i == 0 or (i+1) % config['checkpoint.freq'] == 0:
+        if agent.total_timestep >= int(config['train.timestep']*(checkpoint_count/(config['checkpoint.num'] - 1))):
             agent.checkpoint(logdir, i + 1)
-    agent.checkpoint(logdir, i + 1)
+            checkpoint_count += 1
     pickle_dump(obj=train_logs, f=logdir/'train_logs', ext='.pkl')
     return None
 

baselines/ppo/logs/default/0/config.yml

@@ -2,7 +2,7 @@ ID: 0
 cuda: true
 log.dir: logs/default
 log.freq: 10
-checkpoint.freq: 50
+checkpoint.num: 3
 env.standardize_obs: true
 env.standardize_reward: true
 nn.sizes:
@@ -19,9 +19,6 @@ agent.value_coef: 0.5
 agent.clip_range: 0.2
 env.clip_action: true
 agent.std0: 0.5
-agent.std_style: exp
-agent.std_range: null
-agent.beta: null
 train.timestep: 1000000
 train.timestep_per_iter: 2048
 train.batch_size: 64

baselines/ppo/logs/default/1/1500925526/anim/openaigym.episode_batch.1.347927.stats.json

@@ -0,0 +1 @@
+{"initial_reset_timestamp": 1556979045.49742, "timestamps": [1556979049.2122493], "episode_lengths": [458], "episode_rewards": [1514.5930437670465], "episode_types": ["t", "t"]}

baselines/ppo/logs/default/1/1500925526/anim/openaigym.manifest.1.347927.manifest.json

@@ -0,0 +1 @@
+{"stats": "openaigym.episode_batch.1.347927.stats.json", "videos": [["openaigym.video.1.347927.video000000.mp4", "openaigym.video.1.347927.video000000.meta.json"], ["openaigym.video.1.347927.video000001.mp4", "openaigym.video.1.347927.video000001.meta.json"]], "env_info": {"gym_version": "0.12.1", "env_id": "Hopper-v3"}}

baselines/ppo/logs/default/1/1500925526/anim/openaigym.video.11.1635725.video000000.meta.json → baselines/ppo/logs/default/1/1500925526/anim/openaigym.video.1.347927.video000000.meta.json

@@ -1 +1 @@
-{"episode_id": 0, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil      55. 78.100 / 55. 78.100\\nlibavcodec     57.107.100 / 57.107.100\\nlibavformat    57. 83.100 / 57. 83.100\\nlibavdevice    57. 10.100 / 57. 10.100\\nlibavfilter     6.107.100 /  6.107.100\\nlibavresample   3.  7.  0 /  3.  7.  0\\nlibswscale      4.  8.100 /  4.  8.100\\nlibswresample   2.  9.100 /  2.  9.100\\nlibpostproc    54.  7.100 / 54.  7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "125", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/tmp/lagom/examples/reinforcement_learning/ppo/logs/default/1/1500925526/anim/openaigym.video.11.1635725.video000000.mp4"]}}
+{"episode_id": 0, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil      55. 78.100 / 55. 78.100\\nlibavcodec     57.107.100 / 57.107.100\\nlibavformat    57. 83.100 / 57. 83.100\\nlibavdevice    57. 10.100 / 57. 10.100\\nlibavfilter     6.107.100 /  6.107.100\\nlibavresample   3.  7.  0 /  3.  7.  0\\nlibswscale      4.  8.100 /  4.  8.100\\nlibswresample   2.  9.100 /  2.  9.100\\nlibpostproc    54.  7.100 / 54.  7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "125", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/tmp/lagom/baselines/ppo/logs/default/1/1500925526/anim/openaigym.video.1.347927.video000000.mp4"]}}

baselines/ppo/logs/default/1/1500925526/anim/openaigym.video.11.1635725.video000001.meta.json → baselines/ppo/logs/default/1/1500925526/anim/openaigym.video.1.347927.video000001.meta.json

@@ -1 +1 @@
-{"episode_id": 1, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil      55. 78.100 / 55. 78.100\\nlibavcodec     57.107.100 / 57.107.100\\nlibavformat    57. 83.100 / 57. 83.100\\nlibavdevice    57. 10.100 / 57. 10.100\\nlibavfilter     6.107.100 /  6.107.100\\nlibavresample   3.  7.  0 /  3.  7.  0\\nlibswscale      4.  8.100 /  4.  8.100\\nlibswresample   2.  9.100 /  2.  9.100\\nlibpostproc    54.  7.100 / 54.  7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "125", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/tmp/lagom/examples/reinforcement_learning/ppo/logs/default/1/1500925526/anim/openaigym.video.11.1635725.video000001.mp4"]}}
+{"episode_id": 1, "content_type": "video/mp4", "encoder_version": {"backend": "ffmpeg", "version": "b'ffmpeg version 3.4.4-0ubuntu0.18.04.1 Copyright (c) 2000-2018 the FFmpeg developers\\nbuilt with gcc 7 (Ubuntu 7.3.0-16ubuntu3)\\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.18.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\\nlibavutil      55. 78.100 / 55. 78.100\\nlibavcodec     57.107.100 / 57.107.100\\nlibavformat    57. 83.100 / 57. 83.100\\nlibavdevice    57. 10.100 / 57. 10.100\\nlibavfilter     6.107.100 /  6.107.100\\nlibavresample   3.  7.  0 /  3.  7.  0\\nlibswscale      4.  8.100 /  4.  8.100\\nlibswresample   2.  9.100 /  2.  9.100\\nlibpostproc    54.  7.100 / 54.  7.100\\n'", "cmdline": ["ffmpeg", "-nostats", "-loglevel", "error", "-y", "-r", "125", "-f", "rawvideo", "-s:v", "500x500", "-pix_fmt", "rgb24", "-i", "-", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "/home/zuo/Code/tmp/lagom/baselines/ppo/logs/default/1/1500925526/anim/openaigym.video.1.347927.video000001.mp4"]}}

baselines/ppo/logs/default/1/config.yml

@@ -2,7 +2,7 @@ ID: 1
 cuda: true
 log.dir: logs/default
 log.freq: 10
-checkpoint.freq: 50
+checkpoint.num: 3
 env.standardize_obs: true
 env.standardize_reward: true
 nn.sizes:
@@ -19,9 +19,6 @@ agent.value_coef: 0.5
 agent.clip_range: 0.2
 env.clip_action: true
 agent.std0: 0.5
-agent.std_style: exp
-agent.std_range: null
-agent.beta: null
 train.timestep: 1000000
 train.timestep_per_iter: 2048
 train.batch_size: 64

baselines/ppo/logs/default/2/config.yml

@@ -2,7 +2,7 @@ ID: 2
 cuda: true
 log.dir: logs/default
 log.freq: 10
-checkpoint.freq: 50
+checkpoint.num: 3
 env.standardize_obs: true
 env.standardize_reward: true
 nn.sizes:
@@ -19,9 +19,6 @@ agent.value_coef: 0.5
 agent.clip_range: 0.2
 env.clip_action: true
 agent.std0: 0.5
-agent.std_style: exp
-agent.std_range: null
-agent.beta: null
 train.timestep: 1000000
 train.timestep_per_iter: 2048
 train.batch_size: 64