Merge pull request #174 from zuoxingdong/step_info_trajectory

update ES baselines
zuoxingdong · May 8, 2019 · 9a18e48 · 9a18e48
2 parents 5e2f6b6 + 30034fe
commit 9a18e48
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 24 deletions.
diff --git a/baselines/cmaes/agent.py b/baselines/cmaes/agent.py
@@ -9,6 +9,8 @@
 
 from lagom import BaseAgent
 from lagom.utils import pickle_dump
+from lagom.utils import tensorify
+from lagom.utils import numpify
 from lagom.envs import flatdim
 from lagom.envs.wrappers import get_wrapper
 from lagom.networks import Module
@@ -47,15 +49,14 @@ def __init__(self, config, env, device, **kwargs):
         self.total_timestep = 0
 
     def choose_action(self, obs, **kwargs):
-        if not torch.is_tensor(obs):
-            obs = torch.from_numpy(np.asarray(obs)).float().to(self.device)
+        obs = tensorify(obs, self.device)
         out = {}
         features = self.feature_network(obs)
 
         action_dist = self.action_head(features)
         out['entropy'] = action_dist.entropy()
         action = action_dist.sample()
-        out['raw_action'] = action.detach().cpu().numpy()
+        out['raw_action'] = numpify(action, 'float')
         return out
 
     def learn(self, D, **kwargs):

diff --git a/baselines/cmaes/experiment.py b/baselines/cmaes/experiment.py
@@ -14,6 +14,7 @@
 from lagom.transform import describe
 from lagom.utils import CloudpickleWrapper  # VERY IMPORTANT
 from lagom.utils import pickle_dump
+from lagom.utils import tensorify
 from lagom.utils import set_global_seeds
 from lagom.experiment import Config
 from lagom.experiment import Grid
@@ -30,9 +31,7 @@
 
 
 config = Config(
-    {'cuda': False, 
-     'log.dir': 'logs/default', 
-     'log.freq': 10, 
+    {'log.freq': 10, 
      'checkpoint.num': 3,
 
      'env.id': Grid(['HalfCheetah-v3', 'Hopper-v3', 'Walker2d-v3', 'Swimmer-v3']), 
@@ -42,7 +41,7 @@
 
      # only for continuous control
      'env.clip_action': True,  # clip action within valid bound before step()
-     'agent.std0': 0.5,  # initial std
+     'agent.std0': 0.6,  # initial std
 
      'train.generations': int(1e3),  # total number of ES generations
      'train.popsize': 64,
@@ -76,7 +75,7 @@ def initializer(config, seed, device):
 
 
 def fitness(param):
-    agent.from_vec(torch.from_numpy(param).float())
+    agent.from_vec(tensorify(param, 'cpu'))
     R = []
     H = []
     with torch.no_grad():
@@ -92,9 +91,8 @@ def fitness(param):
     return np.mean(R), np.mean(H)
 
 
-def run(config, seed, device):
+def run(config, seed, device, logdir):
     set_global_seeds(seed)
-    logdir = Path(config['log.dir']) / str(config['ID']) / str(seed)
 
     print('Initializing...')
     agent = Agent(config, make_env(config, seed), device)
@@ -121,7 +119,7 @@ def run(config, seed, device):
             if generation == 0 or (generation+1)%config['log.freq'] == 0:
                 logger.dump(keys=None, index=0, indent=0, border='-'*50)
             if (generation+1) >= int(config['train.generations']*(checkpoint_count/(config['checkpoint.num'] - 1))):
-                agent.from_vec(torch.from_numpy(es.result.xbest).float())
+                agent.from_vec(tensorify(es.result.xbest, 'cpu'))
                 agent.checkpoint(logdir, generation+1)
                 checkpoint_count += 1
     pickle_dump(obj=train_logs, f=logdir/'train_logs', ext='.pkl')
@@ -132,4 +130,8 @@ def run(config, seed, device):
     run_experiment(run=run, 
                    config=config, 
                    seeds=[1770966829, 1500925526, 2054191100], 
-                   num_worker=5)
+                   log_dir='logs/default',
+                   max_workers=None,  # no parallelization 
+                   chunksize=1, 
+                   use_gpu=False,
+                   gpu_ids=None)
diff --git a/baselines/openaies/agent.py b/baselines/openaies/agent.py
@@ -9,6 +9,8 @@
 
 from lagom import BaseAgent
 from lagom.utils import pickle_dump
+from lagom.utils import tensorify
+from lagom.utils import numpify
 from lagom.envs import flatdim
 from lagom.envs.wrappers import get_wrapper
 from lagom.networks import Module
@@ -47,15 +49,14 @@ def __init__(self, config, env, device, **kwargs):
         self.total_timestep = 0
 
     def choose_action(self, obs, **kwargs):
-        if not torch.is_tensor(obs):
-            obs = torch.from_numpy(np.asarray(obs)).float().to(self.device)
+        obs = tensorify(obs, self.device)
         out = {}
         features = self.feature_network(obs)
 
         action_dist = self.action_head(features)
         out['entropy'] = action_dist.entropy()
         action = action_dist.sample()
-        out['raw_action'] = action.detach().cpu().numpy()
+        out['raw_action'] = numpify(action, 'float')
         return out
 
     def learn(self, D, **kwargs):

diff --git a/baselines/openaies/experiment.py b/baselines/openaies/experiment.py
@@ -14,6 +14,7 @@
 from lagom.transform import describe
 from lagom.utils import CloudpickleWrapper  # VERY IMPORTANT
 from lagom.utils import pickle_dump
+from lagom.utils import tensorify
 from lagom.utils import set_global_seeds
 from lagom.experiment import Config
 from lagom.experiment import Grid
@@ -30,9 +31,7 @@
 
 
 config = Config(
-    {'cuda': False, 
-     'log.dir': 'logs/default', 
-     'log.freq': 10, 
+    {'log.freq': 10, 
      'checkpoint.num': 3,
 
      'env.id': Grid(['HalfCheetah-v3', 'Hopper-v3', 'Walker2d-v3', 'Swimmer-v3']), 
@@ -42,7 +41,7 @@
 
      # only for continuous control
      'env.clip_action': True,  # clip action within valid bound before step()
-     'agent.std0': 0.5,  # initial std
+     'agent.std0': 0.6,  # initial std
 
      'train.generations': int(1e3),  # total number of ES generations
      'train.popsize': 64,
@@ -82,7 +81,7 @@ def initializer(config, seed, device):
 
 
 def fitness(param):
-    agent.from_vec(torch.from_numpy(param).float())
+    agent.from_vec(tensorify(param, 'cpu'))
     R = []
     H = []
     with torch.no_grad():
@@ -98,9 +97,8 @@ def fitness(param):
     return np.mean(R), np.mean(H)
 
 
-def run(config, seed, device):
+def run(config, seed, device, logdir):
     set_global_seeds(seed)
-    logdir = Path(config['log.dir']) / str(config['ID']) / str(seed)
 
     print('Initializing...')
     agent = Agent(config, make_env(config, seed), device)
@@ -133,7 +131,7 @@ def run(config, seed, device):
             if generation == 0 or (generation+1)%config['log.freq'] == 0:
                 logger.dump(keys=None, index=0, indent=0, border='-'*50)
             if (generation+1) >= int(config['train.generations']*(checkpoint_count/(config['checkpoint.num'] - 1))):
-                agent.from_vec(torch.from_numpy(es.result.xbest).float())
+                agent.from_vec(tensorify(es.result.xbest, 'cpu'))
                 agent.checkpoint(logdir, generation+1)
                 checkpoint_count += 1
     pickle_dump(obj=train_logs, f=logdir/'train_logs', ext='.pkl')
@@ -144,4 +142,8 @@ def run(config, seed, device):
     run_experiment(run=run, 
                    config=config, 
                    seeds=[1770966829, 1500925526, 2054191100], 
-                   num_worker=5)
+                   log_dir='logs/default',
+                   max_workers=None,  # no parallelization 
+                   chunksize=1, 
+                   use_gpu=False,
+                   gpu_ids=None)