Merge pull request #163 from zuoxingdong/add_ddpg2

minor update
zuoxingdong · May 6, 2019 · 39c6738 · 39c6738
2 parents ebea0eb + 4a712e6
commit 39c6738
Show file tree

Hide file tree

Showing 109 changed files with 36 additions and 237 deletions.
diff --git a/baselines/cem/experiment.py b/baselines/cem/experiment.py
@@ -26,7 +26,7 @@
 from lagom.envs.wrappers import VecStandardizeObservation
 
 from lagom import CEM
-from .agent import Agent
+from baselines.cem.agent import Agent
 
 
 config = Config(

diff --git a/baselines/cmaes/experiment.py b/baselines/cmaes/experiment.py
@@ -26,7 +26,7 @@
 from lagom.envs.wrappers import VecStandardizeObservation
 
 from lagom import CMAES
-from .agent import Agent
+from baselines.cmaes.agent import Agent
 
 
 config = Config(

diff --git a/baselines/ddpg/experiment.py b/baselines/ddpg/experiment.py
@@ -15,12 +15,9 @@
 from lagom.envs.wrappers import ClipAction
 from lagom.envs.wrappers import VecMonitor
 
-from .agent import Agent
-from .engine import Engine
-from .replay_buffer import ReplayBuffer
-# Test for obs/reward normalization
-#from new_engine import Engine
-#from new_replay_buffer import ReplayBuffer
+from baselines.ddpg.agent import Agent
+from baselines.ddpg.engine import Engine
+from baselines.ddpg.replay_buffer import ReplayBuffer
 
 
 config = Config(

diff --git a/baselines/ddpg/new_engine.py b/baselines/ddpg/new_engine.py
diff --git a/baselines/ddpg/new_replay_buffer.py b/baselines/ddpg/new_replay_buffer.py
diff --git a/baselines/openaies/experiment.py b/baselines/openaies/experiment.py
@@ -25,8 +25,8 @@
 from lagom.envs.wrappers import VecMonitor
 from lagom.envs.wrappers import VecStandardizeObservation
 
-from .openaies import OpenAIES
-from .agent import Agent
+from baselines.openaies.openaies import OpenAIES
+from baselines.openaies.agent import Agent
 
 
 config = Config(

diff --git a/baselines/ppo/agent.py b/baselines/ppo/agent.py
@@ -25,7 +25,7 @@
 from lagom.transform import describe
 
 from torch.utils.data import DataLoader
-from .dataset import Dataset
+from baselines.ppo.dataset import Dataset
 
 
 class MLP(Module):

diff --git a/baselines/ppo/experiment.py b/baselines/ppo/experiment.py
@@ -19,8 +19,8 @@
 from lagom.envs.wrappers import VecStandardizeReward
 from lagom.runner import EpisodeRunner
 
-from .agent import Agent
-from .engine import Engine
+from baselines.ppo.agent import Agent
+from baselines.ppo.engine import Engine
 
 
 config = Config(

diff --git a/baselines/td3/experiment.py b/baselines/td3/experiment.py
@@ -15,9 +15,9 @@
 from lagom.envs.wrappers import ClipAction
 from lagom.envs.wrappers import VecMonitor
 
-from .agent import Agent
-from .engine import Engine
-from .replay_buffer import ReplayBuffer
+from baselines.td3.agent import Agent
+from baselines.td3.engine import Engine
+from baselines.td3.replay_buffer import ReplayBuffer
 
 
 config = Config(

diff --git a/baselines/vpg/experiment.py b/baselines/vpg/experiment.py
@@ -19,8 +19,8 @@
 from lagom.envs.wrappers import VecStandardizeReward
 from lagom.runner import EpisodeRunner
 
-from .agent import Agent
-from .engine import Engine
+from baselines.vpg.agent import Agent
+from baselines.vpg.engine import Engine
 
 
 config = Config(
@@ -46,7 +46,7 @@
 
      # only for continuous control
      'env.clip_action': True,  # clip action within valid bound before step()
-     'agent.std0': 0.5,  # initial std
+     'agent.std0': 0.6,  # initial std
 
      'train.timestep': int(1e6),  # total number of training (environmental) timesteps
      'train.timestep_per_iter': 1000,  # number of timesteps per iteration

diff --git a/baselines/vpg/logs/default/0/1500925526/agent_1.pth b/baselines/vpg/logs/default/0/1500925526/agent_1.pth
diff --git a/baselines/vpg/logs/default/0/1500925526/agent_1000.pth b/baselines/vpg/logs/default/0/1500925526/agent_1000.pth
diff --git a/baselines/vpg/logs/default/0/1500925526/agent_500.pth b/baselines/vpg/logs/default/0/1500925526/agent_500.pth
diff --git a/baselines/vpg/logs/default/0/1500925526/obs_moments_1.pth b/baselines/vpg/logs/default/0/1500925526/obs_moments_1.pth
diff --git a/baselines/vpg/logs/default/0/1500925526/obs_moments_1000.pth b/baselines/vpg/logs/default/0/1500925526/obs_moments_1000.pth
diff --git a/baselines/vpg/logs/default/0/1500925526/obs_moments_500.pth b/baselines/vpg/logs/default/0/1500925526/obs_moments_500.pth
diff --git a/baselines/vpg/logs/default/0/1500925526/train_logs.pkl b/baselines/vpg/logs/default/0/1500925526/train_logs.pkl
diff --git a/baselines/vpg/logs/default/0/1770966829/agent_1.pth b/baselines/vpg/logs/default/0/1770966829/agent_1.pth
diff --git a/baselines/vpg/logs/default/0/1770966829/agent_1000.pth b/baselines/vpg/logs/default/0/1770966829/agent_1000.pth
diff --git a/baselines/vpg/logs/default/0/1770966829/agent_500.pth b/baselines/vpg/logs/default/0/1770966829/agent_500.pth
diff --git a/baselines/vpg/logs/default/0/1770966829/obs_moments_1.pth b/baselines/vpg/logs/default/0/1770966829/obs_moments_1.pth
diff --git a/baselines/vpg/logs/default/0/1770966829/obs_moments_1000.pth b/baselines/vpg/logs/default/0/1770966829/obs_moments_1000.pth
diff --git a/baselines/vpg/logs/default/0/1770966829/obs_moments_500.pth b/baselines/vpg/logs/default/0/1770966829/obs_moments_500.pth
diff --git a/baselines/vpg/logs/default/0/1770966829/train_logs.pkl b/baselines/vpg/logs/default/0/1770966829/train_logs.pkl
diff --git a/baselines/vpg/logs/default/0/2054191100/agent_1.pth b/baselines/vpg/logs/default/0/2054191100/agent_1.pth
diff --git a/baselines/vpg/logs/default/0/2054191100/agent_1000.pth b/baselines/vpg/logs/default/0/2054191100/agent_1000.pth
diff --git a/baselines/vpg/logs/default/0/2054191100/agent_500.pth b/baselines/vpg/logs/default/0/2054191100/agent_500.pth
diff --git a/baselines/vpg/logs/default/0/2054191100/obs_moments_1.pth b/baselines/vpg/logs/default/0/2054191100/obs_moments_1.pth
diff --git a/baselines/vpg/logs/default/0/2054191100/obs_moments_1000.pth b/baselines/vpg/logs/default/0/2054191100/obs_moments_1000.pth
diff --git a/baselines/vpg/logs/default/0/2054191100/obs_moments_500.pth b/baselines/vpg/logs/default/0/2054191100/obs_moments_500.pth
diff --git a/baselines/vpg/logs/default/0/2054191100/train_logs.pkl b/baselines/vpg/logs/default/0/2054191100/train_logs.pkl
diff --git a/baselines/vpg/logs/default/0/config.yml b/baselines/vpg/logs/default/0/config.yml
@@ -1,5 +1,5 @@
 ID: 0
-cuda: true
+cuda: false
 log.dir: logs/default
 log.freq: 10
 checkpoint.num: 3
@@ -17,7 +17,7 @@ agent.max_grad_norm: 0.5
 agent.entropy_coef: 0.01
 agent.value_coef: 0.5
 env.clip_action: true
-agent.std0: 0.5
+agent.std0: 0.6
 train.timestep: 1000000
 train.timestep_per_iter: 1000
 env.id: HalfCheetah-v3
diff --git a/baselines/vpg/logs/default/1/1500925526/agent_1.pth b/baselines/vpg/logs/default/1/1500925526/agent_1.pth
diff --git a/baselines/vpg/logs/default/1/1500925526/agent_1000.pth b/baselines/vpg/logs/default/1/1500925526/agent_1000.pth
diff --git a/baselines/vpg/logs/default/1/1500925526/agent_500.pth b/baselines/vpg/logs/default/1/1500925526/agent_500.pth
diff --git a/baselines/vpg/logs/default/1/1500925526/anim/openaigym.episode_batch.0.347927.stats.json b/baselines/vpg/logs/default/1/1500925526/anim/openaigym.episode_batch.0.347927.stats.json
diff --git a/baselines/vpg/logs/default/1/1500925526/anim/openaigym.manifest.0.347927.manifest.json b/baselines/vpg/logs/default/1/1500925526/anim/openaigym.manifest.0.347927.manifest.json
diff --git a/baselines/vpg/logs/default/1/1500925526/anim/openaigym.video.0.347927.video000000.meta.json b/baselines/vpg/logs/default/1/1500925526/anim/openaigym.video.0.347927.video000000.meta.json
diff --git a/baselines/vpg/logs/default/1/1500925526/anim/openaigym.video.0.347927.video000000.mp4 b/baselines/vpg/logs/default/1/1500925526/anim/openaigym.video.0.347927.video000000.mp4
diff --git a/baselines/vpg/logs/default/1/1500925526/anim/openaigym.video.0.347927.video000001.meta.json b/baselines/vpg/logs/default/1/1500925526/anim/openaigym.video.0.347927.video000001.meta.json
diff --git a/baselines/vpg/logs/default/1/1500925526/anim/openaigym.video.0.347927.video000001.mp4 b/baselines/vpg/logs/default/1/1500925526/anim/openaigym.video.0.347927.video000001.mp4
diff --git a/baselines/vpg/logs/default/1/1500925526/obs_moments_1.pth b/baselines/vpg/logs/default/1/1500925526/obs_moments_1.pth
diff --git a/baselines/vpg/logs/default/1/1500925526/obs_moments_1000.pth b/baselines/vpg/logs/default/1/1500925526/obs_moments_1000.pth
diff --git a/baselines/vpg/logs/default/1/1500925526/obs_moments_500.pth b/baselines/vpg/logs/default/1/1500925526/obs_moments_500.pth
diff --git a/baselines/vpg/logs/default/1/1500925526/train_logs.pkl b/baselines/vpg/logs/default/1/1500925526/train_logs.pkl
diff --git a/baselines/vpg/logs/default/1/1770966829/agent_1.pth b/baselines/vpg/logs/default/1/1770966829/agent_1.pth
diff --git a/baselines/vpg/logs/default/1/1770966829/agent_1000.pth b/baselines/vpg/logs/default/1/1770966829/agent_1000.pth
diff --git a/baselines/vpg/logs/default/1/1770966829/agent_500.pth b/baselines/vpg/logs/default/1/1770966829/agent_500.pth
diff --git a/baselines/vpg/logs/default/1/1770966829/obs_moments_1.pth b/baselines/vpg/logs/default/1/1770966829/obs_moments_1.pth
diff --git a/baselines/vpg/logs/default/1/1770966829/obs_moments_1000.pth b/baselines/vpg/logs/default/1/1770966829/obs_moments_1000.pth
diff --git a/baselines/vpg/logs/default/1/1770966829/obs_moments_500.pth b/baselines/vpg/logs/default/1/1770966829/obs_moments_500.pth
diff --git a/baselines/vpg/logs/default/1/1770966829/train_logs.pkl b/baselines/vpg/logs/default/1/1770966829/train_logs.pkl
diff --git a/baselines/vpg/logs/default/1/2054191100/agent_1.pth b/baselines/vpg/logs/default/1/2054191100/agent_1.pth
diff --git a/baselines/vpg/logs/default/1/2054191100/agent_1000.pth b/baselines/vpg/logs/default/1/2054191100/agent_1000.pth
diff --git a/baselines/vpg/logs/default/1/2054191100/agent_500.pth b/baselines/vpg/logs/default/1/2054191100/agent_500.pth
diff --git a/baselines/vpg/logs/default/1/2054191100/obs_moments_1.pth b/baselines/vpg/logs/default/1/2054191100/obs_moments_1.pth
diff --git a/baselines/vpg/logs/default/1/2054191100/obs_moments_1000.pth b/baselines/vpg/logs/default/1/2054191100/obs_moments_1000.pth
diff --git a/baselines/vpg/logs/default/1/2054191100/obs_moments_500.pth b/baselines/vpg/logs/default/1/2054191100/obs_moments_500.pth
diff --git a/baselines/vpg/logs/default/1/2054191100/train_logs.pkl b/baselines/vpg/logs/default/1/2054191100/train_logs.pkl
diff --git a/baselines/vpg/logs/default/1/config.yml b/baselines/vpg/logs/default/1/config.yml
@@ -1,5 +1,5 @@
 ID: 1
-cuda: true
+cuda: false
 log.dir: logs/default
 log.freq: 10
 checkpoint.num: 3
@@ -17,7 +17,7 @@ agent.max_grad_norm: 0.5
 agent.entropy_coef: 0.01
 agent.value_coef: 0.5
 env.clip_action: true
-agent.std0: 0.5
+agent.std0: 0.6
 train.timestep: 1000000
 train.timestep_per_iter: 1000
 env.id: Hopper-v3
diff --git a/baselines/vpg/logs/default/2/1500925526/agent_1.pth b/baselines/vpg/logs/default/2/1500925526/agent_1.pth
diff --git a/baselines/vpg/logs/default/2/1500925526/agent_1000.pth b/baselines/vpg/logs/default/2/1500925526/agent_1000.pth
diff --git a/baselines/vpg/logs/default/2/1500925526/agent_500.pth b/baselines/vpg/logs/default/2/1500925526/agent_500.pth
diff --git a/baselines/vpg/logs/default/2/1500925526/obs_moments_1.pth b/baselines/vpg/logs/default/2/1500925526/obs_moments_1.pth
diff --git a/baselines/vpg/logs/default/2/1500925526/obs_moments_1000.pth b/baselines/vpg/logs/default/2/1500925526/obs_moments_1000.pth
diff --git a/baselines/vpg/logs/default/2/1500925526/obs_moments_500.pth b/baselines/vpg/logs/default/2/1500925526/obs_moments_500.pth
diff --git a/baselines/vpg/logs/default/2/1500925526/train_logs.pkl b/baselines/vpg/logs/default/2/1500925526/train_logs.pkl
diff --git a/baselines/vpg/logs/default/2/1770966829/agent_1.pth b/baselines/vpg/logs/default/2/1770966829/agent_1.pth
diff --git a/baselines/vpg/logs/default/2/1770966829/agent_1000.pth b/baselines/vpg/logs/default/2/1770966829/agent_1000.pth
diff --git a/baselines/vpg/logs/default/2/1770966829/agent_500.pth b/baselines/vpg/logs/default/2/1770966829/agent_500.pth
diff --git a/baselines/vpg/logs/default/2/1770966829/obs_moments_1.pth b/baselines/vpg/logs/default/2/1770966829/obs_moments_1.pth
diff --git a/baselines/vpg/logs/default/2/1770966829/obs_moments_1000.pth b/baselines/vpg/logs/default/2/1770966829/obs_moments_1000.pth
diff --git a/baselines/vpg/logs/default/2/1770966829/obs_moments_500.pth b/baselines/vpg/logs/default/2/1770966829/obs_moments_500.pth
diff --git a/baselines/vpg/logs/default/2/1770966829/train_logs.pkl b/baselines/vpg/logs/default/2/1770966829/train_logs.pkl
diff --git a/baselines/vpg/logs/default/2/2054191100/agent_1.pth b/baselines/vpg/logs/default/2/2054191100/agent_1.pth
diff --git a/baselines/vpg/logs/default/2/2054191100/agent_1000.pth b/baselines/vpg/logs/default/2/2054191100/agent_1000.pth
diff --git a/baselines/vpg/logs/default/2/2054191100/agent_500.pth b/baselines/vpg/logs/default/2/2054191100/agent_500.pth
diff --git a/baselines/vpg/logs/default/2/2054191100/obs_moments_1.pth b/baselines/vpg/logs/default/2/2054191100/obs_moments_1.pth
diff --git a/baselines/vpg/logs/default/2/2054191100/obs_moments_1000.pth b/baselines/vpg/logs/default/2/2054191100/obs_moments_1000.pth
diff --git a/baselines/vpg/logs/default/2/2054191100/obs_moments_500.pth b/baselines/vpg/logs/default/2/2054191100/obs_moments_500.pth
diff --git a/baselines/vpg/logs/default/2/2054191100/train_logs.pkl b/baselines/vpg/logs/default/2/2054191100/train_logs.pkl
diff --git a/baselines/vpg/logs/default/2/config.yml b/baselines/vpg/logs/default/2/config.yml
@@ -1,5 +1,5 @@
 ID: 2
-cuda: true
+cuda: false
 log.dir: logs/default
 log.freq: 10
 checkpoint.num: 3
@@ -17,7 +17,7 @@ agent.max_grad_norm: 0.5
 agent.entropy_coef: 0.01
 agent.value_coef: 0.5
 env.clip_action: true
-agent.std0: 0.5
+agent.std0: 0.6
 train.timestep: 1000000
 train.timestep_per_iter: 1000
 env.id: Walker2d-v3
diff --git a/baselines/vpg/logs/default/3/1500925526/agent_1.pth b/baselines/vpg/logs/default/3/1500925526/agent_1.pth
diff --git a/baselines/vpg/logs/default/3/1500925526/agent_1000.pth b/baselines/vpg/logs/default/3/1500925526/agent_1000.pth
diff --git a/baselines/vpg/logs/default/3/1500925526/agent_500.pth b/baselines/vpg/logs/default/3/1500925526/agent_500.pth
diff --git a/baselines/vpg/logs/default/3/1500925526/obs_moments_1.pth b/baselines/vpg/logs/default/3/1500925526/obs_moments_1.pth
diff --git a/baselines/vpg/logs/default/3/1500925526/obs_moments_1000.pth b/baselines/vpg/logs/default/3/1500925526/obs_moments_1000.pth
diff --git a/baselines/vpg/logs/default/3/1500925526/obs_moments_500.pth b/baselines/vpg/logs/default/3/1500925526/obs_moments_500.pth
diff --git a/baselines/vpg/logs/default/3/1500925526/train_logs.pkl b/baselines/vpg/logs/default/3/1500925526/train_logs.pkl
diff --git a/baselines/vpg/logs/default/3/1770966829/agent_1.pth b/baselines/vpg/logs/default/3/1770966829/agent_1.pth
diff --git a/baselines/vpg/logs/default/3/1770966829/agent_1000.pth b/baselines/vpg/logs/default/3/1770966829/agent_1000.pth
diff --git a/baselines/vpg/logs/default/3/1770966829/agent_500.pth b/baselines/vpg/logs/default/3/1770966829/agent_500.pth
diff --git a/baselines/vpg/logs/default/3/1770966829/obs_moments_1.pth b/baselines/vpg/logs/default/3/1770966829/obs_moments_1.pth
diff --git a/baselines/vpg/logs/default/3/1770966829/obs_moments_1000.pth b/baselines/vpg/logs/default/3/1770966829/obs_moments_1000.pth
diff --git a/baselines/vpg/logs/default/3/1770966829/obs_moments_500.pth b/baselines/vpg/logs/default/3/1770966829/obs_moments_500.pth
diff --git a/baselines/vpg/logs/default/3/1770966829/train_logs.pkl b/baselines/vpg/logs/default/3/1770966829/train_logs.pkl
diff --git a/baselines/vpg/logs/default/3/2054191100/agent_1.pth b/baselines/vpg/logs/default/3/2054191100/agent_1.pth
diff --git a/baselines/vpg/logs/default/3/2054191100/agent_1000.pth b/baselines/vpg/logs/default/3/2054191100/agent_1000.pth
diff --git a/baselines/vpg/logs/default/3/2054191100/agent_500.pth b/baselines/vpg/logs/default/3/2054191100/agent_500.pth
diff --git a/baselines/vpg/logs/default/3/2054191100/obs_moments_1.pth b/baselines/vpg/logs/default/3/2054191100/obs_moments_1.pth
diff --git a/baselines/vpg/logs/default/3/2054191100/obs_moments_1000.pth b/baselines/vpg/logs/default/3/2054191100/obs_moments_1000.pth
diff --git a/baselines/vpg/logs/default/3/2054191100/obs_moments_500.pth b/baselines/vpg/logs/default/3/2054191100/obs_moments_500.pth
diff --git a/baselines/vpg/logs/default/3/2054191100/train_logs.pkl b/baselines/vpg/logs/default/3/2054191100/train_logs.pkl
diff --git a/baselines/vpg/logs/default/3/config.yml b/baselines/vpg/logs/default/3/config.yml
@@ -1,5 +1,5 @@
 ID: 3
-cuda: true
+cuda: false
 log.dir: logs/default
 log.freq: 10
 checkpoint.num: 3
@@ -17,7 +17,7 @@ agent.max_grad_norm: 0.5
 agent.entropy_coef: 0.01
 agent.value_coef: 0.5
 env.clip_action: true
-agent.std0: 0.5
+agent.std0: 0.6
 train.timestep: 1000000
 train.timestep_per_iter: 1000
 env.id: Swimmer-v3
diff --git a/baselines/vpg/logs/default/configs.pkl b/baselines/vpg/logs/default/configs.pkl
diff --git a/baselines/vpg/logs/default/result.png b/baselines/vpg/logs/default/result.png
diff --git a/baselines/vpg/logs/default/source_files/experiment.py b/baselines/vpg/logs/default/source_files/experiment.py
@@ -19,12 +19,12 @@
 from lagom.envs.wrappers import VecStandardizeReward
 from lagom.runner import EpisodeRunner
 
-from agent import Agent
-from engine import Engine
+from baselines.vpg.agent import Agent
+from baselines.vpg.engine import Engine
 
 
 config = Config(
-    {'cuda': True, 
+    {'cuda': False,############True, 
      'log.dir': 'logs/default', 
      'log.freq': 10, 
      'checkpoint.num': 3,
@@ -46,7 +46,7 @@
 
      # only for continuous control
      'env.clip_action': True,  # clip action within valid bound before step()
-     'agent.std0': 0.5,  # initial std
+     'agent.std0': 0.6,  # initial std
 
      'train.timestep': int(1e6),  # total number of training (environmental) timesteps
      'train.timestep_per_iter': 1000,  # number of timesteps per iteration