Better interaction with coverage

.coveragerc

@@ -8,7 +8,7 @@ omit =
     **/strategytests.py
     **/compat*.py
     **/extra/__init__.py
-    **/.tox/*/lib/*/site-packages/hypothesis/internal/branchcheck.py
+    **/.tox/*/lib/*/site-packages/hypothesis/internal/coverage.py
 
 [report]
 exclude_lines =

src/hypothesis/core.py

@@ -47,6 +47,7 @@
 from hypothesis.statistics import note_engine_for_statistics
 from hypothesis.internal.compat import ceil, str_to_bytes, \
     get_type_hints, getfullargspec
+from hypothesis.internal.coverage import IN_COVERAGE_TESTS
 from hypothesis.utils.conventions import infer, not_set
 from hypothesis.internal.escalation import is_hypothesis_file, \
     escalate_hypothesis_internal_error
@@ -62,7 +63,7 @@
 
 try:
     from coverage.tracer import CFileDisposition as FileDisposition
-except ImportError:
+except ImportError:  # pragma: no cover
     from coverage.collector import FileDisposition
 
 
@@ -116,13 +117,13 @@ def reify_and_execute(
     def run(data):
         with BuildContext(data, is_final=is_final):
             orig = sys.gettrace()
-            try:
+            try:  # pragma: no cover
                 sys.settrace(None)
                 import random as rnd_module
                 rnd_module.seed(0)
-                args, kwargs = data.draw(search_strategy)
-            finally:
+            finally:  # pragma: no cover
                 sys.settrace(orig)
+            args, kwargs = data.draw(search_strategy)
 
             if print_example:
                 report(
@@ -523,11 +524,11 @@ def new_given_argspec(original_argspec, generator_kwargs):
 STDLIB = os.path.dirname(os.__file__)
 
 
-def hypothesis_check_include(filename):
+def hypothesis_check_include(filename):  # pragma: no cover
     return filename.endswith('.py')
 
 
-def hypothesis_should_trace(original_filename, frame):
+def hypothesis_should_trace(original_filename, frame):  # pragma: no cover
     disp = FileDisposition()
     assert original_filename is not None
     disp.original_filename = original_filename
@@ -542,7 +543,7 @@ def hypothesis_should_trace(original_filename, frame):
     return disp
 
 
-def escalate_warning(msg):
+def escalate_warning(msg):  # pragma: no cover
     assert False, 'Unexpected warning from coverage: %s' % (msg,)
 
 
@@ -606,7 +607,7 @@ def timed_test(*args, **kwargs):
                 return result
             self.test = timed_test
 
-        if settings.use_coverage:
+        if settings.use_coverage and not IN_COVERAGE_TESTS:  # pragma: no cover
             self.collector = Collector(
                 branch=True,
                 timid=False,
@@ -634,8 +635,8 @@ def evaluate_test_data(self, data):
                 result = self.test_runner(data, reify_and_execute(
                     self.search_strategy, self.test,
                 ))
-            else:
-                assert sys.gettrace() is None
+            else:  # pragma: no cover
+                assert sys.gettrace() is None, sys.gettrace()
                 try:
                     self.collector.data = {}
                     self.collector.start()
@@ -644,6 +645,7 @@ def evaluate_test_data(self, data):
                     ))
                 finally:
                     self.collector.stop()
+                    sys.settrace(None)
                     covdata = CoverageData()
                     self.collector.save_data(covdata)
                     for filename in covdata.measured_files():
@@ -681,31 +683,28 @@ def evaluate_test_data(self, data):
             data.mark_interesting((error_class, filename, lineno))
 
     def run(self):
-        global in_given
-        if in_given:
-            self.collector = None
-            return self._run()
-
-        try:
-            in_given = True
-            self.original_trace = sys.gettrace()
-            sys.settrace(None)
-            return self._run()
-        finally:
-            sys.settrace(self.original_trace)
-            in_given = False
-
-    def _run(self):
         # Tell pytest to omit the body of this function from tracebacks
         __tracebackhide__ = True
         database_key = str_to_bytes(fully_qualified_name(self.test))
         self.start_time = time.time()
+        global in_given
         runner = ConjectureRunner(
             self.evaluate_test_data,
             settings=self.settings, random=self.random,
             database_key=database_key,
         )
-        runner.run()
+
+        if in_given or self.collector is None:
+            runner.run()
+        else:  # pragma: no cover
+            in_given = True
+            self.original_trace = sys.gettrace()
+            try:
+                sys.settrace(None)
+                runner.run()
+                in_given = False
+            finally:
+                sys.settrace(self.original_trace)
         note_engine_for_statistics(runner)
         run_time = time.time() - self.start_time
         timed_out = runner.exit_reason == ExitReason.timeout
@@ -757,6 +756,17 @@ def _run(self):
                         runner.valid_examples,))
 
         if not self.falsifying_examples:
+            if runner.covering_examples:  # pragma: no cover
+                for buf in sorted(set(
+                    v.buffer for v in runner.covering_examples.values()
+                ), key=sort_key):
+                    with self.settings:
+                        self.test_runner(
+                            ConjectureData.for_buffer(buf),
+                            reify_and_execute(
+                                self.search_strategy, self.test,
+                                print_example=False, is_final=False
+                            ))
             return
 
         flaky = 0

src/hypothesis/extra/numpy.py

@@ -26,8 +26,8 @@
 from hypothesis.errors import InvalidArgument
 from hypothesis.searchstrategy import SearchStrategy
 from hypothesis.internal.compat import hrange, text_type
+from hypothesis.internal.coverage import check_function
 from hypothesis.internal.reflection import proxies
-from hypothesis.internal.branchcheck import check_function
 
 TIME_RESOLUTIONS = tuple('Y  M  D  h  m  s  ms  us  ns  ps  fs  as'.split())
 

src/hypothesis/internal/conjecture/data.py

@@ -23,6 +23,7 @@
 from hypothesis.errors import Frozen, InvalidArgument
 from hypothesis.internal.compat import hbytes, hrange, text_type, \
     bit_length, benchmark_time, int_from_bytes, unicode_safe_repr
+from hypothesis.internal.coverage import IN_COVERAGE_TESTS
 
 
 class Status(IntEnum):
@@ -118,7 +119,7 @@ def draw(self, strategy):
 
         if self.depth >= MAX_DEPTH:
             self.mark_invalid()
-        if self.depth == 0:
+        if self.depth == 0 and not IN_COVERAGE_TESTS:  # pragma: no cover
             original_tracer = sys.gettrace()
             if original_tracer is None:
                 return self.__draw(strategy)

src/hypothesis/internal/branchcheck.py → src/hypothesis/internal/coverage.py

@@ -53,7 +53,10 @@ def pretty_file_name(f):
     return result
 
 
-if os.getenv('HYPOTHESIS_INTERNAL_BRANCH_CHECK') == 'true':
+IN_COVERAGE_TESTS = os.getenv('HYPOTHESIS_INTERNAL_COVERAGE') == 'true'
+
+
+if IN_COVERAGE_TESTS:
     log = open('branch-check', 'w')
     written = set()
 

src/hypothesis/internal/escalation.py

@@ -53,5 +53,8 @@ def escalate_hypothesis_internal_error():
         error_type, DeadlineExceeded
     ):
         raise
-    if is_coverage_file(filepath) and issubclass(error_type, AssertionError):
+    # This is so that if we do something wrong and trigger an internal Coverage
+    # error we don't try to catch it. It should be impossible to trigger, but
+    # you never know.
+    if is_coverage_file(filepath):  # pragma: no cover
         raise

src/hypothesis/strategies.py

@@ -34,10 +34,10 @@
     implements_iterator
 from hypothesis.internal.floats import is_negative, float_to_int, \
     int_to_float, count_between_floats
+from hypothesis.internal.coverage import check_function
 from hypothesis.internal.renaming import renamed_arguments
 from hypothesis.utils.conventions import infer, not_set
 from hypothesis.internal.reflection import proxies, required_args
-from hypothesis.internal.branchcheck import check_function
 
 __all__ = [
     'nothing',

tests/common/setup.py

@@ -25,6 +25,7 @@
 from hypothesis.errors import HypothesisDeprecationWarning
 from hypothesis.configuration import set_hypothesis_home_dir
 from hypothesis.internal.charmap import charmap, charmap_file
+from hypothesis.internal.coverage import IN_COVERAGE_TESTS
 
 
 def run():
@@ -51,7 +52,8 @@ def run():
                 v, s.name, s.name, s.default,
             )
 
-    settings.register_profile('default', settings(timeout=unlimited))
+    settings.register_profile('default', settings(
+        timeout=unlimited, use_coverage=not IN_COVERAGE_TESTS))
 
     settings.register_profile(
         'speedy', settings(

tests/cover/test_conjecture_engine.py

@@ -25,8 +25,8 @@
 
 from hypothesis import strategies as st
 from hypothesis import Phase, given, settings, unlimited
-from tests.common.utils import checks_deprecated_behaviour
-from hypothesis.database import ExampleDatabase
+from tests.common.utils import all_values, checks_deprecated_behaviour
+from hypothesis.database import ExampleDatabase, InMemoryExampleDatabase
 from hypothesis.internal.compat import hbytes, hrange, int_from_bytes, \
     bytes_from_list
 from hypothesis.internal.conjecture.data import Status, ConjectureData
@@ -541,3 +541,55 @@ def f(data):
     runner.run()
     assert len(seen) > 256
     assert len({x[0] for x in seen}) == 256
+
+
+def test_will_save_covering_examples():
+    tags = {}
+
+    def tagged(data):
+        b = hbytes(data.draw_bytes(4))
+        try:
+            tag = tags[b]
+        except KeyError:
+            if len(tags) < 10:
+                tag = len(tags)
+                tags[b] = tag
+            else:
+                tag = None
+        if tag is not None:
+            data.add_tag(tag)
+
+    db = InMemoryExampleDatabase()
+    runner = ConjectureRunner(tagged, settings=settings(
+        max_examples=100, max_iterations=10000, max_shrinks=0,
+        buffer_size=1024,
+        database=db,
+    ),  database_key=b'stuff')
+    runner.run()
+    assert len(all_values(db)) == len(tags)
+
+
+def test_will_shrink_covering_examples():
+    seen = [hbytes([255] * 4)]
+
+    def tagged(data):
+        seen[0] = min(seen[0], hbytes(data.draw_bytes(4)))
+        data.add_tag(0)
+
+    db = InMemoryExampleDatabase()
+    runner = ConjectureRunner(tagged, settings=settings(
+        max_examples=100, max_iterations=10000, max_shrinks=0,
+        buffer_size=1024,
+        database=db,
+    ),  database_key=b'stuff')
+    runner.run()
+    assert all_values(db) == set(seen)
+
+
+def test_can_cover_without_a_database_key():
+    def tagged(data):
+        data.add_tag(0)
+
+    runner = ConjectureRunner(tagged, settings=settings(), database_key=None)
+    runner.run()
+    assert len(runner.covering_examples) == 1

tests/nocover/test_coverage.py

@@ -18,9 +18,11 @@
 from __future__ import division, print_function, absolute_import
 
 import hypothesis.strategies as st
+from coverage import Coverage
 from hypothesis import given, settings
 from tests.common.utils import all_values
 from hypothesis.database import InMemoryExampleDatabase
+from hypothesis.internal.compat import hrange
 
 
 def test_tracks_and_saves_coverage():
@@ -41,3 +43,38 @@ def test_branching(i):
     test_branching()
 
     assert len(all_values(db)) == 3
+
+
+def some_function_to_test(a, b, c):
+    result = 0
+    if a:
+        result += 1
+    if b:
+        result += 1
+    if c:
+        result += 1
+    return result
+
+
+LINE_START = some_function_to_test.__code__.co_firstlineno
+with open(__file__) as i:
+    lines = [l.strip() for l in i]
+    LINE_END = LINE_START + lines[LINE_START:].index('')
+
+
+def test_achieves_full_coverage(tmpdir):
+    @given(st.booleans(), st.booleans(), st.booleans())
+    def test(a, b, c):
+        some_function_to_test(a, b, c)
+
+    cov = Coverage(
+        config_file=False, data_file=tmpdir.join('.coveragerc')
+    )
+    cov.start()
+    test()
+    cov.stop()
+
+    data = cov.get_data()
+    lines = data.lines(__file__)
+    for i in hrange(LINE_START + 1, LINE_END + 1):
+        assert i in lines

tox.ini

@@ -141,7 +141,7 @@ deps =
     -rrequirements/test.txt
     -rrequirements/coverage.txt
 setenv=
-    HYPOTHESIS_INTERNAL_BRANCH_CHECK=true
+    HYPOTHESIS_INTERNAL_COVERAGE=true
 commands =
     python -m coverage --version
     python -m coverage debug sys