Merge pull request #2126 from pckroon/fix_1996

Implement pytest.mark.parametrize detection. Fix #1996

azure-pipelines.yml

@@ -46,8 +46,8 @@ jobs:
           TASK: check-py27-typing
         check-nose:
           TASK: check-nose
-        check-pytest30:
-          TASK: check-pytest30
+        check-pytest43:
+          TASK: check-pytest43
         check-django22:
           TASK: check-django22
         check-django21:

hypothesis-python/RELEASE.rst

@@ -0,0 +1,12 @@
+RELEASE_TYPE: minor
+
+This release fixes a bug where our example database logic did not distinguish 
+between failing examples based on arguments from a ``@pytest.mark.parametrize(...)``.
+This could in theory cause data loss if a common failure overwrote a rare one, and
+in practice caused occasional file-access collisions in highly concurrent workloads
+(e.g. during a 300-way parametrize on 16 cores).
+
+For internal reasons this also involves bumping the minimum supported version of
+:pypi:`pytest` to 4.3
+
+Thanks to Peter C Kroon for the Hacktoberfest patch!

hypothesis-python/setup.py

@@ -58,7 +58,7 @@ def local_file(name):
     "lark": ["lark-parser>=0.6.5"],
     "numpy": ["numpy>=1.9.0"],
     "pandas": ["pandas>=0.19"],
-    "pytest": ["pytest>=3.0"],
+    "pytest": ["pytest>=4.3"],
     "dpcontracts": ["dpcontracts>=0.4"],
     # We only support Django versions with upstream support - see
     # https://www.djangoproject.com/download/#supported-versions

hypothesis-python/src/hypothesis/extra/pytestplugin.py

@@ -115,6 +115,12 @@ def pytest_runtest_call(item):
     if not (hasattr(item, "obj") and is_hypothesis_test(item.obj)):
         yield
     else:
+        if item.get_closest_marker("parametrize") is not None:
+            # Give every parametrized test invocation a unique database key
+            item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = item.nodeid.encode(
+                "utf-8"
+            )
+
         store = StoringReporter(item.config)
 
         def note_statistics(stats):

hypothesis-python/tests/pytest/test_parametrized_db_keys.py

@@ -0,0 +1,44 @@
+# coding=utf-8
+#
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Most of this work is copyright (C) 2013-2019 David R. MacIver
+# (david@drmaciver.com), but it contains contributions by others. See
+# CONTRIBUTING.rst for a full list of people who may hold copyright, and
+# consult the git log if you need to determine who owns an individual
+# contribution.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.
+#
+# END HEADER
+
+from __future__ import absolute_import, division, print_function
+
+DB_KEY_TESTCASE = """
+from hypothesis import settings, given
+from hypothesis.database import InMemoryExampleDatabase
+from hypothesis.strategies import booleans
+import pytest
+
+DB = InMemoryExampleDatabase()
+
+
+@settings(database=DB)
+@given(booleans())
+@pytest.mark.parametrize("hi", (1, 2, 3))
+@pytest.mark.xfail()
+def test_dummy_for_parametrized_db_keys(hi, i):
+    assert Fail  # Test *must* fail for it to end up the database anyway
+
+
+def test_DB_keys_for_parametrized_test():
+    assert len(DB.data) == 3
+"""
+
+
+def test_db_keys_for_parametrized_tests_are_unique(testdir):
+    script = testdir.makepyfile(DB_KEY_TESTCASE)
+    testdir.runpytest(script).assert_outcomes(xfailed=3, passed=1)

hypothesis-python/tox.ini

@@ -148,11 +148,11 @@ deps =
 commands=
     nosetests tests/cover/test_testdecorators.py
 
-[testenv:pytest30]
+[testenv:pytest43]
 deps =
     -r../requirements/test.txt
 commands=
-    pip install pytest==3.0 pytest-xdist==1.24 pytest-forked==0.2
+    pip install pytest==4.3 pytest-xdist==1.25 pytest-forked==0.2
     python -m pytest tests/pytest tests/cover/test_testdecorators.py
 
 

tooling/src/hypothesistooling/__main__.py

@@ -460,7 +460,7 @@ def standard_tox_task(name):
 
 
 standard_tox_task("nose")
-standard_tox_task("pytest30")
+standard_tox_task("pytest43")
 
 for n in [20, 21, 22, 111]:
     standard_tox_task("django%d" % (n,))