[WIP] add support for seeding environments (#135)

* Make environments seedable

* Fix monitor bugs

- Set monitor_id before setting the infix. This was a bug that would yield incorrect results with multiple monitors.
- Remove extra pid from stats recorder filename. This should be purely cosmetic.

* Start uploading seeds in episode_batch

* Fix _bigint_from_bytes for python3

* Set seed explicitly in random_agent

* Pass through seed argument

* Also pass through random state to spaces

* Pass random state into the observation/action spaces

* Make all _seed methods return the list of used seeds

* Switch over to np.random where possible

* Start hashing seeds, and also seed doom engine

* Fixup seeding determinism in many cases

* Seed before loading the ROM

* Make seeding more Python3 friendly

* Make the MuJoCo skipping a bit more forgiving

* Remove debugging PDB calls

* Make setInt argument into raw bytes

* Validate and upload seeds

* Skip box2d

* Make seeds smaller, and change representation of seeds in upload

* Handle long seeds

* Fix RandomAgent example to be deterministic

* Handle integer types correctly in Python2 and Python3

* Try caching pip

* Try adding swap

* Add df and free calls

* Bump swap

* Bump swap size

* Try setting overcommit

* Try other sysctls

* Try fixing overcommit

* Try just setting overcommit_memory=1

* Add explanatory comment

* Add what's new section to readme

* BUG: Mark ElevatorAction-ram-v0 as non-deterministic for now

* Document seed

* Move nondetermistic check into spec

.gitignore

@@ -23,8 +23,6 @@ ghostdriver.log

junk
MUJOCO_LOG.txt
mujoco-bundle


rllab_mujoco

@@ -36,3 +34,4 @@ tutorial/*.html

# PyCharm project files
.idea
vizdoom.ini

.travis.yml

@@ -2,7 +2,7 @@ dist: trusty
sudo: required
cache:
  apt: true
  pip: false
  pip: true
language: python
addons:
  apt:
@@ -30,6 +30,11 @@ before_install:
    # In a pull request, there are no secrets, and hence no MuJoCo:
    # https://docs.travis-ci.com/user/pull-requests#Security-Restrictions-when-testing-Pull-Requests.
  - '[ -z ${MUJOCO_KEY_BUNDLE+x} ] || ( curl https://openai-public.s3-us-west-2.amazonaws.com/mujoco/$MUJOCO_KEY_BUNDLE.tar.gz | tar xz -C ~/.mujoco )'
  # Without this line, Travis has fork()s fail with an out of memory
  # error. (These fork()s are for spawning the subprocess for video
  # recording.) We should debug the memory usage at some stage, but
  # simply setting overcommit is a good starting point.
  - sudo sysctl -w vm.overcommit_memory=1
env:
  - DISPLAY=:12
install: pip install tox-travis

README.rst

@@ -251,3 +251,10 @@ We are using `nose2 <https://github.com/nose-devs/nose2>`_ for tests. You can ru
	  nose2

You can also run tests in a specific directory by using the ``-s`` option, or by passing in the specific name of the test. See the `nose2 docs <http://nose2.readthedocs.org/en/latest/usage.html#naming-tests>`_ for more details.

What's new
----------

- 2016-05-28: For controlled reproducibility, envs now support seeding
  (cf #91 and #135). The monitor records which seeds are used. We will
  soon add seed information to the display on the scoreboard.

examples/agents/random_agent.py

@@ -19,14 +19,18 @@ def act(self, observation, reward, done):
    logger.setLevel(logging.INFO)

    env = gym.make('CartPole-v0' if len(sys.argv)<2 else sys.argv[1])
    agent = RandomAgent(env.action_space)

    # You provide the directory to write to (can be an existing
    # directory, including one with existing data -- all monitor files
    # will be namespaced). You can also dump to a tempdir if you'd
    # like: tempfile.mkdtemp().
    outdir = '/tmp/random-agent-results'
    env.monitor.start(outdir, force=True)
    env.monitor.start(outdir, force=True, seed=0)

    # This declaration must go *after* the monitor call, since the
    # monitor's seeding creates a new action_space instance with the
    # appropriate pseudorandom number generator.
    agent = RandomAgent(env.action_space)

    episode_count = 100
    max_steps = 200

gym/configuration.py

@@ -1,9 +1,4 @@
import hashlib
import numpy as np
import logging
import os
import random
import struct
import sys

import gym
@@ -40,48 +35,3 @@ def undo_logger_setup():
    root_logger.removeHandler(handler)
    gym.logger.setLevel(logging.NOTSET)
    requests_logger.setLevel(logging.NOTSET)

def seed(a=None):
    """Seeds the 'random' and 'numpy.random' generators. By default,
    Python seeds these with the system time. Call this if you are
    using multiple processes.

    Notes:
        SECURITY SENSITIVE: a bug here would allow people to generate fake results. Please let us know if you find one :).

    Args:
        a (Optional[int, str]): None or no argument seeds from an operating system specific randomness source. If an int or str passed, then all of bits are used.
    """
    # Adapted from https://svn.python.org/projects/python/tags/r32/Lib/random.py
    if a is None:
        a = bigint_from_bytes(os.urandom(32))

    if isinstance(a, str):
        a = a.encode('utf8')
        a += hashlib.sha512(a).digest()
        a = bigint_from_bytes(a)

    # Actually seed the generators
    random.seed(a)
    np.random.seed(int_list_from_bigint(a))

    return a

# TODO: don't hardcode sizeof_int here
def bigint_from_bytes(bytes):
    sizeof_int = 4
    padding = sizeof_int - len(bytes) % sizeof_int
    bytes += '\0' * padding
    int_count = len(bytes) / sizeof_int
    unpacked = struct.unpack("{}I".format(int_count), bytes)
    accum = 0
    for i, val in enumerate(unpacked):
        accum += 2 ** (sizeof_int * 8 * i) * val
    return accum

def int_list_from_bigint(bigint):
    ints = []
    while bigint > 0:
        bigint, mod = divmod(bigint, 2 ** 32)
        ints.append(mod)
    return ints

gym/core.py

@@ -17,17 +17,20 @@ class Env(object):

    The main API methods that users of this class need to know are:

        reset
        step
        reset
        render
        close
        seed

    When implementing an environment, override the following methods
    in your subclass:

        _step
        _reset
        _render
        _close
        _seed

    And set the following attributes:

@@ -70,6 +73,7 @@ def _render(self, mode='human', close=False):
        if close:
            return
        raise NotImplementedError
    def _seed(self, seed=None): return []

    @property
    def monitor(self):
@@ -172,7 +176,9 @@ def close(self):
        Environments will automatically close() themselves when
        garbage collected or when the program exits.
        """
        if self._closed:
        # _closed will be missing if this instance is still
        # initializing.
        if not hasattr(self, '_closed') or self._closed:
            return

        self._close()
@@ -181,6 +187,23 @@ def close(self):
        # end up with double close.
        self._closed = True

    def seed(self, seed=None):
        """Sets the seed for this env's random number generator(s).

        Note:
            Some environments use multiple pseudorandom number generators.
            We want to capture all such seeds used in order to ensure that
            there aren't accidental correlations between multiple generators.

        Returns:
            list<bigint>: Returns the list of seeds used in this env's random
              number generators. The first value in the list should be the
              "main" seed, or the value which a reproducer should pass to
              'seed'. Often, the main seed equals the provided 'seed', but
              this won't be true if seed=None, for example.
        """
        return self._seed(seed)

    def __del__(self):
        self.close()

gym/envs/__init__.py

@@ -228,11 +228,21 @@
        name = ''.join([g.capitalize() for g in game.split('_')])
        if obs_type == 'ram':
            name = '{}-ram'.format(name)

        nondeterministic = False
        if game == 'elevator_action' and obs_type == 'ram':
            # ElevatorAction-ram-v0 seems to yield slightly
            # non-deterministic observations about 10% of the time. We
            # should track this down eventually, but for now we just
            # mark it as nondetermistic.
            nondeterministic = True

        register(
            id='{}-v0'.format(name),
            entry_point='gym.envs.atari:AtariEnv',
            kwargs={'game': game, 'obs_type': obs_type},
            timestep_limit=10000,
            nondeterministic=nondeterministic,
        )

# Board games
@@ -248,6 +258,11 @@
        'illegal_move_mode': 'lose',
        'board_size': 9,
    },
    # The pachi player seems not to be determistic given a fixed seed.
    # (Reproduce by running 'import gym; h = gym.make('Go9x9-v0'); h.seed(1); h.reset(); h.step(15); h.step(16); h.step(17)' a few times.)
    #
    # This is probably due to a computation time limit.
    nondetermistic=True,
)

register(
@@ -260,6 +275,7 @@
        'illegal_move_mode': 'lose',
        'board_size': 19,
    },
    nondetermistic=True,
)

register(

gym/envs/algorithmic/algorithmic_env.py

@@ -1,8 +1,7 @@
from gym import Env
from gym.spaces import Discrete, Tuple
from gym.utils import colorize
from gym.utils import colorize, seeding
import numpy as np
import random
from six import StringIO
import sys
import math
@@ -17,6 +16,7 @@ class AlgorithmicEnv(Env):

    def __init__(self, inp_dim=1, base=10, chars=False):
        global hash_base

        hash_base = 50 ** np.arange(inp_dim)
        self.base = base
        self.last = 10
@@ -27,10 +27,17 @@ def __init__(self, inp_dim=1, base=10, chars=False):
        self.inp_dim = inp_dim
        AlgorithmicEnv.current_length = 2
        tape_control = []
        self.action_space = Tuple(([Discrete(2 * inp_dim), Discrete(2), Discrete(self.base)]))
        self.observation_space = Discrete(self.base + 1)

        self._seed()
        self.reset()

    def _seed(self, seed=None):
        self.np_random, seed = seeding.np_random(seed)

        self.action_space = Tuple(([Discrete(2 * self.inp_dim, np_random=self.np_random), Discrete(2, np_random=self.np_random), Discrete(self.base, np_random=self.np_random)]))
        self.observation_space = Discrete(self.base + 1, np_random=self.np_random)
        return [seed]

    def _get_obs(self, pos=None):
        if pos is None:
            pos = self.x
@@ -198,6 +205,6 @@ def _reset(self):
            AlgorithmicEnv.sum_rewards = []
        self.sum_reward = 0.0
        self.time = 0
        self.total_len = random.randrange(3) + AlgorithmicEnv.current_length
        self.total_len = self.np_random.randint(3) + AlgorithmicEnv.current_length
        self.set_data()
        return self._get_obs()

gym/envs/algorithmic/copy.py

@@ -2,7 +2,6 @@
Task is to copy content from the input tape to
the output tape. http://arxiv.org/abs/1511.07275
"""
import random
import numpy as np
from gym.envs.algorithmic import algorithmic_env
from gym.envs.algorithmic.algorithmic_env import ha
@@ -17,8 +16,7 @@ def set_data(self):
        self.content = {}
        self.target = {}
        for i in range(self.total_len):
            val = random.randrange(self.base)
            val = self.np_random.randint(self.base)
            self.content[ha(np.array([i]))] = val
            self.target[i] = val
        self.total_reward = self.total_len 

        self.total_reward = self.total_len

gym/envs/algorithmic/duplicated_input.py

@@ -3,7 +3,6 @@
http://arxiv.org/abs/1511.07275
"""

import random
import numpy as np
from gym.envs.algorithmic import algorithmic_env
from gym.envs.algorithmic.algorithmic_env import ha
@@ -20,7 +19,7 @@ def set_data(self):
        self.target = {}
        copies = int(self.total_len / self.duplication)
        for i in range(copies):
            val = random.randrange(self.base)
            val = self.np_random.randint(self.base)
            self.target[i] = val
            for d in range(self.duplication):
                self.content[ha(np.array([i * self.duplication + d]))] = val

gym/envs/algorithmic/repeat_copy.py

@@ -2,7 +2,6 @@
Task is to copy content multiple-times from the input tape to
the output tape. http://arxiv.org/abs/1511.07275
"""
import random
import numpy as np
from gym.envs.algorithmic import algorithmic_env
from gym.envs.algorithmic.algorithmic_env import ha
@@ -20,10 +19,9 @@ def set_data(self):
        self.target = {}
        unique = set()
        for i in range(self.total_len):
            val = random.randrange(self.base)
            val = self.np_random.randint(self.base)
            self.content[ha(np.array([i]))] = val
            self.target[i] = val
            self.target[2 * self.total_len - i - 1] = val
            self.target[2 * self.total_len + i] = val
        self.total_reward = 3.0 * self.total_len + 0.9

gym/envs/algorithmic/reverse.py

@@ -3,7 +3,6 @@
http://arxiv.org/abs/1511.07275
"""

import random
import numpy as np
from gym.envs.algorithmic import algorithmic_env
from gym.envs.algorithmic.algorithmic_env import ha
@@ -21,7 +20,7 @@ def set_data(self):
        self.content = {}
        self.target = {}
        for i in range(self.total_len):
            val = random.randrange(self.base)
            val = self.np_random.randint(self.base)
            self.content[ha(np.array([i]))] = val
            self.target[self.total_len - i - 1] = val
        self.total_reward = self.total_len + 0.9

gym/envs/algorithmic/reversed_addition.py

@@ -1,4 +1,3 @@
import random
import numpy as np
from gym.envs.algorithmic import algorithmic_env
from gym.envs.algorithmic.algorithmic_env import ha
@@ -17,14 +16,12 @@ def set_data(self):
        for i in range(self.total_len):
            vals = []
            for k in range(self.rows):
                val = random.randrange(self.base)
                val = self.np_random.randint(self.base)
                self.content[ha(np.array([i, k]))] = val
                vals.append(val)
            total = sum(vals) + curry
            self.target[i] = total % self.base
            curry = total / self.base
        if curry > 0:
            self.target[self.total_len] = curry
        self.total_reward = self.total_len 


        self.total_reward = self.total_len