[WIP] add support for seeding environments (#135)

* Make environments seedable

* Fix monitor bugs

- Set monitor_id before setting the infix. This was a bug that would yield incorrect results with multiple monitors.
- Remove extra pid from stats recorder filename. This should be purely cosmetic.

* Start uploading seeds in episode_batch

* Fix _bigint_from_bytes for python3

* Set seed explicitly in random_agent

* Pass through seed argument

* Also pass through random state to spaces

* Pass random state into the observation/action spaces

* Make all _seed methods return the list of used seeds

* Switch over to np.random where possible

* Start hashing seeds, and also seed doom engine

* Fixup seeding determinism in many cases

* Seed before loading the ROM

* Make seeding more Python3 friendly

* Make the MuJoCo skipping a bit more forgiving

* Remove debugging PDB calls

* Make setInt argument into raw bytes

* Validate and upload seeds

* Skip box2d

* Make seeds smaller, and change representation of seeds in upload

* Handle long seeds

* Fix RandomAgent example to be deterministic

* Handle integer types correctly in Python2 and Python3

* Try caching pip

* Try adding swap

* Add df and free calls

* Bump swap

* Bump swap size

* Try setting overcommit

* Try other sysctls

* Try fixing overcommit

* Try just setting overcommit_memory=1

* Add explanatory comment

* Add what's new section to readme

* BUG: Mark ElevatorAction-ram-v0 as non-deterministic for now

* Document seed

* Move nondetermistic check into spec

.gitignore

@@ -23,8 +23,6 @@ ghostdriver.log
 
 junk
 MUJOCO_LOG.txt
-mujoco-bundle
-
 
 rllab_mujoco
 
@@ -36,3 +34,4 @@ tutorial/*.html
 
 # PyCharm project files
 .idea
+vizdoom.ini

.travis.yml

@@ -2,7 +2,7 @@ dist: trusty
 sudo: required
 cache:
   apt: true
-  pip: false
+  pip: true
 language: python
 addons:
   apt:
@@ -30,6 +30,11 @@ before_install:
     # In a pull request, there are no secrets, and hence no MuJoCo:
     # https://docs.travis-ci.com/user/pull-requests#Security-Restrictions-when-testing-Pull-Requests.
   - '[ -z ${MUJOCO_KEY_BUNDLE+x} ] || ( curl https://openai-public.s3-us-west-2.amazonaws.com/mujoco/$MUJOCO_KEY_BUNDLE.tar.gz | tar xz -C ~/.mujoco )'
+  # Without this line, Travis has fork()s fail with an out of memory
+  # error. (These fork()s are for spawning the subprocess for video
+  # recording.) We should debug the memory usage at some stage, but
+  # simply setting overcommit is a good starting point.
+  - sudo sysctl -w vm.overcommit_memory=1
 env:
   - DISPLAY=:12
 install: pip install tox-travis

README.rst

@@ -251,3 +251,10 @@ We are using `nose2 <https://github.com/nose-devs/nose2>`_ for tests. You can ru
 	  nose2
 
 You can also run tests in a specific directory by using the ``-s`` option, or by passing in the specific name of the test. See the `nose2 docs <http://nose2.readthedocs.org/en/latest/usage.html#naming-tests>`_ for more details.
+
+What's new
+----------
+
+- 2016-05-28: For controlled reproducibility, envs now support seeding
+  (cf #91 and #135). The monitor records which seeds are used. We will
+  soon add seed information to the display on the scoreboard.

examples/agents/random_agent.py

@@ -19,14 +19,18 @@ def act(self, observation, reward, done):
     logger.setLevel(logging.INFO)
 
     env = gym.make('CartPole-v0' if len(sys.argv)<2 else sys.argv[1])
-    agent = RandomAgent(env.action_space)
 
     # You provide the directory to write to (can be an existing
     # directory, including one with existing data -- all monitor files
     # will be namespaced). You can also dump to a tempdir if you'd
     # like: tempfile.mkdtemp().
     outdir = '/tmp/random-agent-results'
-    env.monitor.start(outdir, force=True)
+    env.monitor.start(outdir, force=True, seed=0)
+
+    # This declaration must go *after* the monitor call, since the
+    # monitor's seeding creates a new action_space instance with the
+    # appropriate pseudorandom number generator.
+    agent = RandomAgent(env.action_space)
 
     episode_count = 100
     max_steps = 200

gym/configuration.py

@@ -1,9 +1,4 @@
-import hashlib
-import numpy as np
 import logging
-import os
-import random
-import struct
 import sys
 
 import gym
@@ -40,48 +35,3 @@ def undo_logger_setup():
     root_logger.removeHandler(handler)
     gym.logger.setLevel(logging.NOTSET)
     requests_logger.setLevel(logging.NOTSET)
-
-def seed(a=None):
-    """Seeds the 'random' and 'numpy.random' generators. By default,
-    Python seeds these with the system time. Call this if you are
-    using multiple processes.
-
-    Notes:
-        SECURITY SENSITIVE: a bug here would allow people to generate fake results. Please let us know if you find one :).
-
-    Args:
-        a (Optional[int, str]): None or no argument seeds from an operating system specific randomness source. If an int or str passed, then all of bits are used.
-    """
-    # Adapted from https://svn.python.org/projects/python/tags/r32/Lib/random.py
-    if a is None:
-        a = bigint_from_bytes(os.urandom(32))
-
-    if isinstance(a, str):
-        a = a.encode('utf8')
-        a += hashlib.sha512(a).digest()
-        a = bigint_from_bytes(a)
-
-    # Actually seed the generators
-    random.seed(a)
-    np.random.seed(int_list_from_bigint(a))
-
-    return a
-
-# TODO: don't hardcode sizeof_int here
-def bigint_from_bytes(bytes):
-    sizeof_int = 4
-    padding = sizeof_int - len(bytes) % sizeof_int
-    bytes += '\0' * padding
-    int_count = len(bytes) / sizeof_int
-    unpacked = struct.unpack("{}I".format(int_count), bytes)
-    accum = 0
-    for i, val in enumerate(unpacked):
-        accum += 2 ** (sizeof_int * 8 * i) * val
-    return accum
-
-def int_list_from_bigint(bigint):
-    ints = []
-    while bigint > 0:
-        bigint, mod = divmod(bigint, 2 ** 32)
-        ints.append(mod)
-    return ints

gym/core.py

@@ -17,17 +17,20 @@ class Env(object):
 
     The main API methods that users of this class need to know are:
 
-        reset
         step
+        reset
         render
         close
+        seed
 
     When implementing an environment, override the following methods
     in your subclass:
 
         _step
         _reset
         _render
+        _close
+        _seed
 
     And set the following attributes:
 
@@ -70,6 +73,7 @@ def _render(self, mode='human', close=False):
         if close:
             return
         raise NotImplementedError
+    def _seed(self, seed=None): return []
 
     @property
     def monitor(self):
@@ -172,7 +176,9 @@ def close(self):
         Environments will automatically close() themselves when
         garbage collected or when the program exits.
         """
-        if self._closed:
+        # _closed will be missing if this instance is still
+        # initializing.
+        if not hasattr(self, '_closed') or self._closed:
             return
 
         self._close()
@@ -181,6 +187,23 @@ def close(self):
         # end up with double close.
         self._closed = True
 
+    def seed(self, seed=None):
+        """Sets the seed for this env's random number generator(s).
+
+        Note:
+            Some environments use multiple pseudorandom number generators.
+            We want to capture all such seeds used in order to ensure that
+            there aren't accidental correlations between multiple generators.
+
+        Returns:
+            list<bigint>: Returns the list of seeds used in this env's random
+              number generators. The first value in the list should be the
+              "main" seed, or the value which a reproducer should pass to
+              'seed'. Often, the main seed equals the provided 'seed', but
+              this won't be true if seed=None, for example.
+        """
+        return self._seed(seed)
+
     def __del__(self):
         self.close()
 

gym/envs/__init__.py

@@ -228,11 +228,21 @@
         name = ''.join([g.capitalize() for g in game.split('_')])
         if obs_type == 'ram':
             name = '{}-ram'.format(name)
+
+        nondeterministic = False
+        if game == 'elevator_action' and obs_type == 'ram':
+            # ElevatorAction-ram-v0 seems to yield slightly
+            # non-deterministic observations about 10% of the time. We
+            # should track this down eventually, but for now we just
+            # mark it as nondetermistic.
+            nondeterministic = True
+
         register(
             id='{}-v0'.format(name),
             entry_point='gym.envs.atari:AtariEnv',
             kwargs={'game': game, 'obs_type': obs_type},
             timestep_limit=10000,
+            nondeterministic=nondeterministic,
         )
 
 # Board games
@@ -248,6 +258,11 @@
         'illegal_move_mode': 'lose',
         'board_size': 9,
     },
+    # The pachi player seems not to be determistic given a fixed seed.
+    # (Reproduce by running 'import gym; h = gym.make('Go9x9-v0'); h.seed(1); h.reset(); h.step(15); h.step(16); h.step(17)' a few times.)
+    #
+    # This is probably due to a computation time limit.
+    nondetermistic=True,
 )
 
 register(
@@ -260,6 +275,7 @@
         'illegal_move_mode': 'lose',
         'board_size': 19,
     },
+    nondetermistic=True,
 )
 
 register(

gym/envs/algorithmic/algorithmic_env.py

@@ -1,8 +1,7 @@
 from gym import Env
 from gym.spaces import Discrete, Tuple
-from gym.utils import colorize
+from gym.utils import colorize, seeding
 import numpy as np
-import random
 from six import StringIO
 import sys
 import math
@@ -17,6 +16,7 @@ class AlgorithmicEnv(Env):
 
     def __init__(self, inp_dim=1, base=10, chars=False):
         global hash_base
+
         hash_base = 50 ** np.arange(inp_dim)
         self.base = base
         self.last = 10
@@ -27,10 +27,17 @@ def __init__(self, inp_dim=1, base=10, chars=False):
         self.inp_dim = inp_dim
         AlgorithmicEnv.current_length = 2
         tape_control = []
-        self.action_space = Tuple(([Discrete(2 * inp_dim), Discrete(2), Discrete(self.base)]))
-        self.observation_space = Discrete(self.base + 1)
+
+        self._seed()
         self.reset()
 
+    def _seed(self, seed=None):
+        self.np_random, seed = seeding.np_random(seed)
+
+        self.action_space = Tuple(([Discrete(2 * self.inp_dim, np_random=self.np_random), Discrete(2, np_random=self.np_random), Discrete(self.base, np_random=self.np_random)]))
+        self.observation_space = Discrete(self.base + 1, np_random=self.np_random)
+        return [seed]
+
     def _get_obs(self, pos=None):
         if pos is None:
             pos = self.x
@@ -198,6 +205,6 @@ def _reset(self):
             AlgorithmicEnv.sum_rewards = []
         self.sum_reward = 0.0
         self.time = 0
-        self.total_len = random.randrange(3) + AlgorithmicEnv.current_length
+        self.total_len = self.np_random.randint(3) + AlgorithmicEnv.current_length
         self.set_data()
         return self._get_obs()

gym/envs/algorithmic/copy.py

@@ -2,7 +2,6 @@
 Task is to copy content from the input tape to
 the output tape. http://arxiv.org/abs/1511.07275
 """
-import random
 import numpy as np
 from gym.envs.algorithmic import algorithmic_env
 from gym.envs.algorithmic.algorithmic_env import ha
@@ -17,8 +16,7 @@ def set_data(self):
         self.content = {}
         self.target = {}
         for i in range(self.total_len):
-            val = random.randrange(self.base)
+            val = self.np_random.randint(self.base)
             self.content[ha(np.array([i]))] = val
             self.target[i] = val
-        self.total_reward = self.total_len 
-
+        self.total_reward = self.total_len

gym/envs/algorithmic/duplicated_input.py

@@ -3,7 +3,6 @@
 http://arxiv.org/abs/1511.07275
 """
 
-import random
 import numpy as np
 from gym.envs.algorithmic import algorithmic_env
 from gym.envs.algorithmic.algorithmic_env import ha
@@ -20,7 +19,7 @@ def set_data(self):
         self.target = {}
         copies = int(self.total_len / self.duplication)
         for i in range(copies):
-            val = random.randrange(self.base)
+            val = self.np_random.randint(self.base)
             self.target[i] = val
             for d in range(self.duplication):
                 self.content[ha(np.array([i * self.duplication + d]))] = val

gym/envs/algorithmic/repeat_copy.py

@@ -2,7 +2,6 @@
 Task is to copy content multiple-times from the input tape to
 the output tape. http://arxiv.org/abs/1511.07275
 """
-import random
 import numpy as np
 from gym.envs.algorithmic import algorithmic_env
 from gym.envs.algorithmic.algorithmic_env import ha
@@ -20,10 +19,9 @@ def set_data(self):
         self.target = {}
         unique = set()
         for i in range(self.total_len):
-            val = random.randrange(self.base)
+            val = self.np_random.randint(self.base)
             self.content[ha(np.array([i]))] = val
             self.target[i] = val
             self.target[2 * self.total_len - i - 1] = val
             self.target[2 * self.total_len + i] = val
         self.total_reward = 3.0 * self.total_len + 0.9
-

gym/envs/algorithmic/reverse.py

@@ -3,7 +3,6 @@
 http://arxiv.org/abs/1511.07275
 """
 
-import random
 import numpy as np
 from gym.envs.algorithmic import algorithmic_env
 from gym.envs.algorithmic.algorithmic_env import ha
@@ -21,7 +20,7 @@ def set_data(self):
         self.content = {}
         self.target = {}
         for i in range(self.total_len):
-            val = random.randrange(self.base)
+            val = self.np_random.randint(self.base)
             self.content[ha(np.array([i]))] = val
             self.target[self.total_len - i - 1] = val
         self.total_reward = self.total_len + 0.9

gym/envs/algorithmic/reversed_addition.py

@@ -1,4 +1,3 @@
-import random
 import numpy as np
 from gym.envs.algorithmic import algorithmic_env
 from gym.envs.algorithmic.algorithmic_env import ha
@@ -17,14 +16,12 @@ def set_data(self):
         for i in range(self.total_len):
             vals = []
             for k in range(self.rows):
-                val = random.randrange(self.base)
+                val = self.np_random.randint(self.base)
                 self.content[ha(np.array([i, k]))] = val
                 vals.append(val)
             total = sum(vals) + curry
             self.target[i] = total % self.base
             curry = total / self.base
         if curry > 0:
             self.target[self.total_len] = curry
-        self.total_reward = self.total_len 
-
-
+        self.total_reward = self.total_len