Merge pull request #154 from Bachmann1234/profiles

Create profile loading mechanism

.gitignore

@@ -6,3 +6,6 @@ venv*
 .hypothesis
 docs/_build
 *.egg-info
+_build
+.tox
+.coverage

CONTRIBUTING.rst

@@ -111,7 +111,7 @@ For example, it's super useful and higly appreciated if you do any of:
 * Build libraries and tools on top of Hypothesis outside the main repo
 
 Of, if you're OK with the pull request but don't feel quite ready to touch the code, you can always
-help to improve the documentation. Spot a tyop? Fix it up and send me a pull request! 
+help to improve the documentation. Spot a tyop? Fix it up and send me a pull request!
 
 If you need any help with any of these, get in touch and I'll be extremely happy to provide it.
 
@@ -135,6 +135,7 @@ their individual contributions.
 * `kbara <https://www.github.com/kbara>`_
 * `marekventur <https://www.github.com/marekventur>`_
 * `Marius Gedminas <https://www.github.com/mgedmin>`_ (`marius@gedmin.as <mailto:marius@gedmin.as>`_)
+* `Matt Bachmann <https://www.github.com/bachmann1234>`_ (`bachmann.matt@gmail.com <mailto:bachmann.matt@gmail.com>`_)
 * `Nicholas Chammas <https://www.github.com/nchammas>`_
 * `Richard Boulton <https://www.github.com/rboulton>`_ (`richard@tartarus.org <mailto:richard@tartarus.org>`_)
 * `Saul Shanabrook <https://www.github.com/saulshanabrook>`_ (`s.shanabrook@gmail.com <mailto:s.shanabrook@gmail.com>`_)

docs/details.rst

@@ -285,6 +285,64 @@ You can also override the default by setting the environment variable
 setting ``HYPOTHESIS_VERBOSITY_LEVEL=verbose`` will run all your tests printing
 intermediate results and errors.
 
+.. _settings_profiles:
+
+~~~~~~~~~~~~~~~~~
+Settings Profiles
+~~~~~~~~~~~~~~~~~
+
+Depending on your environment you may want different default settings.
+For example: during development you may want to lower the number of examples
+to speed up the tests. However, in a CI environment you may want more examples
+so you are more likely to find bugs.
+
+Hypothesis allows you to define different settings profiles. These profiles
+can be loaded at any time.
+
+Loading a profile changes the default settings but will not change the behavior
+of tests that explicitly change the settings.
+
+.. code:: python
+
+    >>> from hypothesis import Settings
+    >>> Settings.register_profile("ci", Settings(max_examples=1000))
+    >>> Settings().max_examples
+    200
+    >>> Settings.load_profile("ci")
+    >>> Settings().max_examples
+    1000
+
+Instead of loading the profile and overriding the defaults you can retrieve profiles for
+specific tests.
+
+.. code:: python
+
+  >>> with Settings.get_profile("ci"):
+  ...     print(Settings().max_examples)
+  ...
+  1000
+
+Optionally, you may define the environment variable to load a profile for you.
+This is the suggested pattern for running your tests on CI.
+The code below should run in a `conftest.py` or any setup/initialization section of your test suite.
+If this variable is not defined the Hypothesis defined defaults will be loaded.
+
+.. code:: python
+
+    >>> from hypothesis import Settings
+    >>> Settings.register_profile("ci", Settings(max_examples=1000))
+    >>> Settings.register_profile("dev", Settings(max_examples=10))
+    >>> Settings.register_profile("debug", Settings(max_examples=10, verbosity=Verbosity.verbose))
+    >>> Settings.load_profile(os.getenv(u'HYPOTHESIS_PROFILE', 'default'))
+
+If you are using the hypothesis pytest plugin. If the profile was registered
+while loading in a conftest file you can load it with the command line
+option ``--hypothesis-profile``
+
+.. code:: bash
+
+    $ py.test tests --hypothesis-profile <profile-name>
+
 ---------------------
 Defining strategies
 ---------------------

docs/extras.rst

@@ -161,7 +161,7 @@ in as a providers argument:
     >>> class KittenProvider(BaseProvider):
     ...     def meows(self):
     ...             return 'meow %d' % (self.random_number(digits=10),)
-    ... 
+    ...
     >>> fake_factory('meows', providers=[KittenProvider]).example()
     'meow 9139348419'
 
@@ -192,5 +192,6 @@ package will remain for compatibility reasons if it does.
 
 hypothesis-pytest is the world's most basic pytest plugin. Install it to get
 slightly better integrated example reporting when using @given and running
-under pytest. That's basically all it does.
+under pytest.
 
+It can also load :ref:`Settings Profiles <settings_profiles>`.

src/hypothesis/extra/pytestplugin.py

@@ -19,6 +19,9 @@
 import pytest
 
 PYTEST_VERSION = tuple(map(int, pytest.__version__.split('.')[:3]))
+LOAD_PROFILE_OPTION = '--hypothesis-profile'
+
+PYTEST_VERSION = tuple(map(int, pytest.__version__.split('.')))
 if PYTEST_VERSION >= (2, 7, 0):
     class StoringReporter(object):
 
@@ -31,6 +34,19 @@ def __call__(self, msg):
                 print(msg)
             self.results.append(msg)
 
+    def pytest_addoption(parser):
+        parser.addoption(
+            LOAD_PROFILE_OPTION,
+            action='store',
+            help='Load in a registered hypothesis settings profile'
+        )
+
+    def pytest_configure(config):
+        from hypothesis import settings
+        profile = config.getoption(LOAD_PROFILE_OPTION)
+        if profile:
+            settings.Settings.load_profile(profile)
+
     @pytest.mark.hookwrapper
     def pytest_pyfunc_call(pyfuncitem):
         from hypothesis.reporting import with_reporter

src/hypothesis/settings.py

@@ -24,6 +24,7 @@
 from __future__ import division, print_function, absolute_import
 
 import os
+import copy
 import inspect
 import threading
 from collections import namedtuple
@@ -139,6 +140,8 @@ class Settings(SettingsMeta('Settings', (object,), {})):
 
     """
 
+    _profiles = {}
+
     def __getattr__(self, name):
         if name in all_settings:
             d = all_settings[name].default
@@ -198,6 +201,10 @@ def define_setting(cls, name, description, default, options=None):
         all_settings[name] = Setting(
             name, description.strip(), default, options)
         setattr(cls, name, SettingsProperty(name))
+        if cls.default:
+            setattr(cls.default, name, default)
+        for profile in cls._profiles.values():
+            setattr(profile, name, default)
 
     def __setattr__(self, name, value):
         if name in all_settings:
@@ -259,6 +266,48 @@ def __exit__(self, *args, **kwargs):
         default_context_manager = self.defaults_stack().pop()
         return default_context_manager.__exit__(*args, **kwargs)
 
+    @staticmethod
+    def register_profile(name, settings):
+        """registers a collection of values to be used as a settings profile.
+        These settings can be loaded in by name. Enable different defaults for
+        different settings.
+
+        - settings is a Settings object
+
+        """
+        Settings._profiles[name] = copy.copy(settings)
+
+    @staticmethod
+    def get_profile(name):
+        """Return the profile with the given name.
+
+        - name is a string representing the name of the profile
+         to load
+        A InvalidArgument exception will be thrown if the
+         profile does not exist
+
+        """
+        try:
+            return copy.copy(Settings._profiles[name])
+        except KeyError:
+            raise InvalidArgument(
+                "Profile '{0}' has not been registered".format(
+                    name
+                )
+            )
+
+    @staticmethod
+    def load_profile(name):
+        """Loads in the settings defined in the profile provided If the profile
+        does not exist an InvalidArgument will be thrown.
+
+        Any setting not defined in the profile will be the library
+        defined default for that setting
+
+        """
+        Settings.default = Settings.get_profile(name)
+
+
 Setting = namedtuple(
     u'Setting', (u'name', u'description', u'default', u'options'))
 
@@ -390,8 +439,6 @@ def by_name(cls, key):
             return result
         raise InvalidArgument(u'No such verbosity level %r' % (key,))
 
-Settings.default = Settings()
-
 Verbosity.quiet = Verbosity(u'quiet', 0)
 Verbosity.normal = Verbosity(u'normal', 1)
 Verbosity.verbose = Verbosity(u'verbose', 2)
@@ -414,3 +461,6 @@ def by_name(cls, key):
     default=DEFAULT_VERBOSITY,
     description=u'Control the verbosity level of Hypothesis messages',
 )
+
+Settings.register_profile('default', Settings())
+Settings.load_profile('default')

tests/cover/test_settings.py

@@ -17,6 +17,7 @@
 from __future__ import division, print_function, absolute_import
 
 import pytest
+import hypothesis
 from hypothesis.errors import InvalidArgument
 from hypothesis.database import ExampleDatabase
 from hypothesis.settings import Settings, Verbosity
@@ -148,8 +149,82 @@ def test_can_assign_database(db):
 
 
 def test_can_assign_default_settings():
-    Settings.default = Settings(max_examples=1100)
-    assert Settings.default.max_examples == 1100
-    with Settings(max_examples=10):
-        assert Settings.default.max_examples == 10
-    assert Settings.default.max_examples == 1100
+    try:
+        Settings.default = Settings(max_examples=1100)
+        assert Settings.default.max_examples == 1100
+        with Settings(max_examples=10):
+            assert Settings.default.max_examples == 10
+        assert Settings.default.max_examples == 1100
+    finally:
+        # Reset settings.default to default when settings
+        # is first loaded
+        Settings.default = Settings(max_examples=200)
+
+
+def test_load_profile():
+    Settings.load_profile('default')
+    assert Settings.default.max_examples == 200
+    assert Settings.default.max_shrinks == 500
+    assert Settings.default.min_satisfying_examples == 5
+
+    Settings.register_profile(
+        'test',
+        Settings(
+            max_examples=10,
+            max_shrinks=5
+        )
+    )
+
+    Settings.load_profile('test')
+
+    assert Settings.default.max_examples == 10
+    assert Settings.default.max_shrinks == 5
+    assert Settings.default.min_satisfying_examples == 5
+
+    Settings.load_profile('default')
+
+    assert Settings.default.max_examples == 200
+    assert Settings.default.max_shrinks == 500
+    assert Settings.default.min_satisfying_examples == 5
+
+
+def test_loading_profile_resets_defaults():
+    assert Settings.default.min_satisfying_examples == 5
+    Settings.default.min_satisfying_examples = 100
+    assert Settings.default.min_satisfying_examples == 100
+    Settings.load_profile('default')
+    assert Settings.default.min_satisfying_examples == 5
+
+
+def test_loading_profile_keeps_expected_behaviour():
+    Settings.register_profile('ci', Settings(max_examples=10000))
+    Settings.load_profile('ci')
+    assert Settings().max_examples == 10000
+    with Settings(max_examples=5):
+        assert Settings().max_examples == 5
+    assert Settings().max_examples == 10000
+
+
+def test_modifying_registered_profile_does_not_change_profile():
+    ci_profile = Settings(max_examples=10000)
+    Settings.register_profile('ci', ci_profile)
+    ci_profile.max_examples = 1
+    Settings.load_profile('ci')
+    assert Settings().max_examples == 10000
+
+
+def test_load_non_existent_profile():
+    with pytest.raises(hypothesis.errors.InvalidArgument):
+        Settings.get_profile('nonsense')
+
+
+def test_define_setting_then_loading_profile():
+    x = Settings()
+    Settings.define_setting(
+        u'fun_times',
+        default=3, description=u'Something something spoon',
+        options=(1, 2, 3, 4),
+    )
+    Settings.register_profile('hi', Settings(fun_times=2))
+    assert x.fun_times == 3
+    assert Settings.get_profile('hi').fun_times == 2

tests/pytest/test_profiles.py

@@ -0,0 +1,43 @@
+# coding=utf-8
+
+# This file is part of Hypothesis (https://github.com/DRMacIver/hypothesis)
+
+# Most of this work is copyright (C) 2013-2015 David R. MacIver
+# (david@drmaciver.com), but it contains contributions by others. See
+# https://github.com/DRMacIver/hypothesis/blob/master/CONTRIBUTING.rst for a
+# full list of people who may hold copyright, and consult the git log if you
+# need to determine who owns an individual contribution.
+
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at http://mozilla.org/MPL/2.0/.
+
+# END HEADER
+
+from __future__ import division, print_function, absolute_import
+
+from hypothesis.extra.pytestplugin import LOAD_PROFILE_OPTION
+
+pytest_plugins = str('pytester')
+
+CONFTEST = """
+from hypothesis.settings import Settings
+Settings.register_profile("test", Settings(max_examples=1))
+"""
+
+TESTSUITE = """
+from hypothesis import given
+from hypothesis.strategies import integers
+from hypothesis.settings import Settings
+
+def test_this_one_is_ok():
+    assert Settings().max_examples == 1
+"""
+
+
+def test_runs_reporting_hook(testdir):
+    script = testdir.makepyfile(TESTSUITE)
+    testdir.makeconftest(CONFTEST)
+    result = testdir.runpytest(script, LOAD_PROFILE_OPTION, 'test')
+    out = '\n'.join(result.stdout.lines)
+    assert '1 passed' in out