Fix loading of charmap from cache file (#2166)

Fix loading of charmap from cache file

CONTRIBUTING.rst

@@ -261,6 +261,7 @@ their individual contributions.
 * `Peadar Coyle <https://github.com/springcoil>`_ (peadarcoyle@gmail.com)
 * `Pierre-Jean Campigotto <https://github.com/PJCampi>`_
 * `Richard Boulton <https://www.github.com/rboulton>`_ (richard@tartarus.org)
+* `Robert Knight <https://github.com/robertknight>`_ (robertknight@gmail.com)
 * `Ryan Soklaski <https://www.github.com/rsokl>`_ (rsoklaski@gmail.com)
 * `Ryan Turner <https://github.com/rdturnermtl>`_ (ryan.turner@uber.com)
 * `Sam Bishop (TechDragon) <https://github.com/techdragon>`_ (sam@techdragon.io)

hypothesis-python/RELEASE.rst

@@ -0,0 +1,7 @@
+RELEASE_TYPE: patch
+
+This release fixes a bug (:issue:`2166`) where a Unicode character info
+cache file was generated but never used on subsequent test runs, causing tests
+to run more slowly than they should have.
+
+Thanks to Robert Knight for this bugfix!

hypothesis-python/src/hypothesis/internal/charmap.py

@@ -59,17 +59,25 @@ def charmap():
         f = charmap_file()
         try:
             with gzip.GzipFile(f, "rb") as i:
-                tmp_charmap = dict(json.loads(i))
+                # When the minimum Python 3 version becomes 3.6, this can be
+                # simplified to `json.load(i)` without needing to decode first.
+                data = i.read().decode()
+                tmp_charmap = dict(json.loads(data))
 
         except Exception:
+            # This loop is reduced to using only local variables for performance;
+            # indexing and updating containers is a ~3x slowdown.  This doesn't fix
+            # https://github.com/HypothesisWorks/hypothesis/issues/2108 but it helps.
+            category = unicodedata.category  # Local variable -> ~20% speedup!
             tmp_charmap = {}
-            for i in range(0, sys.maxunicode + 1):
-                cat = unicodedata.category(hunichr(i))
-                rs = tmp_charmap.setdefault(cat, [])
-                if rs and rs[-1][-1] == i - 1:
-                    rs[-1][-1] += 1
-                else:
-                    rs.append([i, i])
+            last_cat = category(hunichr(0))
+            last_start = 0
+            for i in range(1, sys.maxunicode + 1):
+                cat = category(hunichr(i))
+                if cat != last_cat:
+                    tmp_charmap.setdefault(last_cat, []).append([last_start, i - 1])
+                    last_cat, last_start = cat, i
+            tmp_charmap.setdefault(last_cat, []).append([last_start, sys.maxunicode])
 
             try:
                 # Write the Unicode table atomically

hypothesis-python/tests/cover/test_charmap.py

@@ -20,6 +20,7 @@
 import os
 import sys
 import tempfile
+import time
 import unicodedata
 
 import hypothesis.internal.charmap as cm
@@ -109,6 +110,22 @@ def test_recreate_charmap():
     assert x == y
 
 
+def test_uses_cached_charmap():
+    cm.charmap()
+
+    # Reset the last-modified time of the cache file to a point in the past.
+    mtime = int(time.time() - 1000)
+    os.utime(cm.charmap_file(), (mtime, mtime))
+    statinfo = os.stat(cm.charmap_file())
+    assert statinfo.st_mtime == mtime
+
+    # Force reload of charmap from cache file and check that mtime is unchanged.
+    cm._charmap = None
+    cm.charmap()
+    statinfo = os.stat(cm.charmap_file())
+    assert statinfo.st_mtime == mtime
+
+
 def test_union_empty():
     assert cm._union_intervals([], []) == ()
     assert cm._union_intervals([], [[1, 2]]) == ((1, 2),)

hypothesis-python/tests/cover/test_deadline.py

@@ -69,7 +69,7 @@ def test_flaky_slow(i):
 
 
 def test_deadlines_participate_in_shrinking():
-    @settings(deadline=500)
+    @settings(deadline=500, max_examples=1000)
     @given(st.integers(min_value=0))
     def slow_if_large(i):
         if i >= 1000: