Merge ee35194 into cc173e7

.coveragerc

@@ -7,3 +7,4 @@ relative_files = True
 exclude_lines =
     pragma: no cover
     class I[A-Z]\w+\((Interface|I[A-Z].*)\):
+    self.fail

.github/workflows/tests.yml

@@ -52,6 +52,11 @@ env:
   PYTHONDEVMODE: 1
   PYTHONFAULTHANDLER: 1
   ZOPE_INTERFACE_STRICT_IRO: 1
+  # Require C extensions; this will be disabled later for
+  # PyPy because zope.interface through at least 5.3
+  # tries to import its C module even on PyPy with this, but
+  # the wheels tend not to have it.
+  PURE_PYTHON: 0
 
   PIP_UPGRADE_STRATEGY: eager
   # Don't get warnings about Python 2 support being deprecated. We
@@ -132,7 +137,12 @@ jobs:
         run: |
           pip install -U pip
           pip install -U setuptools wheel twine
-          pip install -U coveralls coverage
+          pip install -U coveralls coverage persistent
+
+      - name: Set pure Python
+        if: ${{ startsWith(matrix.python-version, 'pypy') }}
+        run: |
+          echo PURE_PYTHON=1 >> $GITHUB_ENV
 
       - name: Build BTrees
         run: |
@@ -198,6 +208,10 @@ jobs:
           key: ${{ runner.os }}-pip-${{ matrix.python-version }}
           restore-keys: |
             ${{ runner.os }}-pip-
+      - name: Set pure Python
+        if: ${{ startsWith(matrix.python-version, 'pypy') }}
+        run: |
+          echo PURE_PYTHON=1 >> $GITHUB_ENV
 
       - name: Download BTrees wheel
         uses: actions/download-artifact@v2

.manylinux-install.sh

@@ -32,6 +32,8 @@ else
     export CFLAGS="-Ofast $CFLAGS"
 fi
 
+export PURE_PYTHON=0
+
 # Compile wheels
 cd /io/
 for variant in `ls -d /opt/python/cp{27,36,37,38,39}*`; do
@@ -49,10 +51,6 @@ for variant in `ls -d /opt/python/cp{27,36,37,38,39}*`; do
         # in just one interpreter, the newest one on the list (which in principle
         # should be the fastest), and we don't install the ZODB extra.
         if [[ "${PYBIN}" == *"cp39"* ]]; then
-            # Until we move from ./BTrees/ to ./src/BTrees/,
-            # installing BTrees as non-editable is incompatible with using
-            # /io/ as the working directory: the local directory shadows the installed
-            # version, and we can't import the C extensions.
             "${PYBIN}/pip" install -e .[test]
             "${PYBIN}/python" -c 'import BTrees.OOBTree; print(BTrees.OOBTree.BTree, BTrees.OOBTree.BTreePy)'
             "${PYBIN}/python" -m unittest discover -s src

CHANGES.rst

@@ -21,6 +21,14 @@
 
 - Build and upload aarch64 wheels.
 
+- Make a value of ``0`` in the ``PURE_PYTHON`` environment variable
+  require the C extensions (except on PyPy). Previously, and if this
+  variable is unset, missing or unusable C extensions would be
+  silently ignored. With this variable set to ``0``, an
+  ``ImportError`` will be raised if the C extensions are unavailable.
+  See `issue 156
+  <https://github.com/zopefoundation/BTrees/issues/156>`_.
+
 4.7.2 (2020-04-07)
 ==================
 

src/BTrees/_compat.py

@@ -15,13 +15,9 @@
 import sys
 
 PYPY = hasattr(sys, 'pypy_version_info')
-# We can and do build the C extensions on PyPy, but
-# as of Persistent 4.2.5 the persistent C extension is not
-# built on PyPy, so importing our C extension will fail anyway.
-PURE_PYTHON = os.environ.get('PURE_PYTHON', PYPY)
 
 
-if sys.version_info[0] < 3: #pragma NO COVER Python2
+if sys.version_info[0] < 3: # pragma: no cover Python2
 
     PY2 = True
     PY3 = False
@@ -43,7 +39,7 @@ def compare(x, y):
     def _ascii(x):
         return bytes(x)
 
-else: #pragma NO COVER Python3
+else: # pragma: no cover Python3
 
     PY2 = False
     PY3 = True
@@ -67,18 +63,80 @@ def _ascii(x):
         return bytes(x, 'ascii')
 
 
-def import_c_extension(mod_globals):
+def _c_optimizations_required():
+    """
+    Return a true value if the C optimizations are required.
+
+    This uses the ``PURE_PYTHON`` variable as documented in `import_c_extension`.
+    """
+    pure_env = os.environ.get('PURE_PYTHON')
+    require_c = pure_env == "0"
+    return require_c
+
+
+def _c_optimizations_available(module_name):
+    """
+    Return the C optimization module, if available, otherwise
+    a false value.
+
+    If the optimizations are required but not available, this
+    raises the ImportError.
+
+    This does not say whether they should be used or not.
+    """
     import importlib
+    catch = () if _c_optimizations_required() else (ImportError,)
+    try:
+        return importlib.import_module('BTrees._' + module_name)
+    except catch: # pragma: no cover
+        return False
+
+
+def _c_optimizations_ignored():
+    """
+    The opposite of `_c_optimizations_required`.
+    """
+    pure_env = os.environ.get('PURE_PYTHON')
+    return pure_env != "0" if pure_env is not None else PYPY
+
+
+def _should_attempt_c_optimizations():
+    """
+    Return a true value if we should attempt to use the C optimizations.
+
+    This takes into account whether we're on PyPy and the value of the
+    ``PURE_PYTHON`` environment variable, as defined in `import_c_extension`.
+    """
+    if PYPY:
+        return False
+
+    if _c_optimizations_required():
+        return True
+    return not _c_optimizations_ignored()
+
+
+def import_c_extension(mod_globals):
+    """
+    Call this function with the globals of a module that implements
+    Python versions of a BTree family to find the C optimizations.
+
+    If the ``PURE_PYTHON`` environment variable is set to any value
+    other than ``"0"``, or we're on PyPy, ignore the C implementation.
+    If the C implementation cannot be imported, return the Python
+    version. If ``PURE_PYTHON`` is set to ``"0"``, *require* the C
+    implementation (let the ImportError propagate); the exception again
+    is PyPy, where we never use the C extension (although it builds here, the
+    ``persistent`` library doesn't provide native extensions for PyPy).
+
+    """
     c_module = None
     module_name = mod_globals['__name__']
     assert module_name.startswith('BTrees.')
     module_name = module_name.split('.')[1]
-    if not PURE_PYTHON:
-        try:
-            c_module = importlib.import_module('BTrees._' + module_name)
-        except ImportError:
-            pass
-    if c_module is not None:
+    if _should_attempt_c_optimizations():
+        c_module = _c_optimizations_available(module_name)
+
+    if c_module:
         new_values = dict(c_module.__dict__)
         new_values.pop("__name__", None)
         new_values.pop('__file__', None)

src/BTrees/check.py

@@ -404,7 +404,7 @@ def complain(self, msg, obj, path):
                 ".".join(map(str, path)))
         self.errors.append(s)
 
-class Printer(Walker): #pragma NO COVER
+class Printer(Walker): # pragma: no cover
     def __init__(self, obj):
         Walker.__init__(self, obj)
 
@@ -458,6 +458,6 @@ def check(btree):
 
     Checker(btree).check()
 
-def display(btree): #pragma NO COVER
+def display(btree): # pragma: no cover
     "Display the internal structure of a BTree, Bucket, TreeSet or Set."
     Printer(btree).display()

src/BTrees/tests/common.py

@@ -21,7 +21,7 @@
 
 
 from BTrees._compat import PY3
-from BTrees._compat import PURE_PYTHON
+from BTrees._compat import _c_optimizations_ignored
 from BTrees._compat import PYPY
 from BTrees._base   import _tp_name
 
@@ -45,13 +45,13 @@ def _no_op(test_method):
 else:
     _skip_on_32_bits = _no_op
 
-if PURE_PYTHON:
+if _c_optimizations_ignored():
     skipOnPurePython = skip("Not on Pure Python")
 else:
     skipOnPurePython = _no_op
 
 def _skip_if_pure_py_and_py_test(self):
-    if PURE_PYTHON and 'Py' in type(self).__name__:
+    if _c_optimizations_ignored() and 'Py' in type(self).__name__:
         # No need to run this again. The "C" tests will catch it.
         # This relies on the fact that we always define tests in pairs,
         # one normal/C and one with Py in the name for the Py test.
@@ -121,7 +121,7 @@ def _closeRoot(self, root):
         transaction.abort()
         root._p_jar.close()
 
-    
+
 class Base(ZODBAccess, SignedMixin):
     # Tests common to all types: sets, buckets, and BTrees
 
@@ -165,7 +165,7 @@ def testPurePython(self):
         self.assertNotIn('_', module_name)
         self.assertIs(getattr(module, class_name), kind)
 
-        if not PURE_PYTHON and 'Py' not in type(self).__name__:
+        if not _c_optimizations_ignored() and 'Py' not in type(self).__name__:
             self.assertIsNot(getattr(module, class_name + 'Py'), kind)
 
     @_skip_wo_ZODB
@@ -1624,21 +1624,21 @@ def testSetstateBadChild(self):
         # t.__getstate__(), or t[0]=1 corrupt memory and crash.
         with self.assertRaises(TypeError) as exc:
             t.__setstate__(
-                    (
-                        (xchild,), # child0 is neither tree nor bucket
-                        b
-                    )
+                (
+                    (xchild,), # child0 is neither tree nor bucket
+                    b
                 )
+            )
         self.assertEqual(str(exc.exception), typeErrOK)
 
         # if the following is allowed, e.g. t[5]=1 corrupts memory and crash.
         with self.assertRaises(TypeError) as exc:
             t.__setstate__(
-                    (
-                        (b, 4, xchild),
-                        b
-                    )
+                (
+                    (b, 4, xchild),
+                    b
                 )
+            )
         self.assertEqual(str(exc.exception), typeErrOK)
 
 
@@ -2590,7 +2590,7 @@ def testValuesIgnored(self):
 
     def testBigInput(self):
         N = 100000
-        if (PURE_PYTHON or 'Py' in type(self).__name__) and not PYPY:
+        if (_c_optimizations_ignored() or 'Py' in type(self).__name__) and not PYPY:
             # This is extremely slow in CPython implemented in Python,
             # taking 20s or more on a 2015-era laptop
             N = N // 10

src/BTrees/tests/test_Length.py

@@ -67,9 +67,9 @@ def test_change_overflows_to_long(self):
         import sys
         try:
             length = self._makeOne(sys.maxint)
-        except AttributeError: #pragma NO COVER Py3k
+        except AttributeError: # pragma: no cover Py3k
             return
-        else: #pragma NO COVER Py2
+        else: # pragma: no cover Py2
             self.assertEqual(length(), sys.maxint)
             self.assertTrue(type(length()) is int)
             length.change(+1)
@@ -80,9 +80,9 @@ def test_change_underflows_to_long(self):
         import sys
         try:
             minint = (-sys.maxint) - 1
-        except AttributeError: #pragma NO COVER Py3k
+        except AttributeError: # pragma: no cover Py3k
             return
-        else: #pragma NO COVER Py2
+        else: # pragma: no cover Py2
             length = self._makeOne(minint)
             self.assertEqual(length(), minint)
             self.assertTrue(type(length()) is int)

src/BTrees/tests/test_utils.py

@@ -36,7 +36,7 @@ def test_w_big_positive(self):
         import sys
         try:
             self.assertEqual(self._callFUT(sys.maxint), sys.maxint)
-        except AttributeError: #pragma NO COVER Py3k
+        except AttributeError: # pragma: no cover Py3k
             pass
 
 

src/BTrees/utils.py

@@ -24,7 +24,7 @@ def non_negative(int_val):
     return int_val
 
 
-def positive_id(obj): #pragma NO COVER
+def positive_id(obj): # pragma: no cover
     """Return id(obj) as a non-negative integer."""
     return non_negative(id(obj))
 

tox.ini

@@ -10,14 +10,15 @@ usedevelop = true
 extras =
     test
 commands =
-    # Temporary work around. Avoid zope.testrunner pending
-    # https://github.com/zopefoundation/zope.security/issues/71
-    python -m unittest discover -s src {posargs}
+    python -m zope.testrunner --test-path=src --auto-color --auto-progress {posargs}
 setenv =
     PYTHONFAULTHANDLER=1
     PYTHONDEVMODE=1
     ZOPE_INTERFACE_STRICT_IRO=1
+    PURE_PYTHON=0
     pure: PURE_PYTHON=1
+    pypy: PURE_PYTHON=1
+    pypy3: PURE_PYTHON=1
 
 #[testenv:jython]
 #commands =
@@ -37,19 +38,17 @@ deps =
 
 [testenv:coverage]
 basepython =
-    python3.6
+    python3
 commands =
-    coverage run -m zope.testrunner --test-path=. --auto-color --auto-progress []
+    coverage run -m zope.testrunner --test-path=src --auto-color --auto-progress {posargs}
     coverage report --fail-under=92
 deps =
     coverage
-# Temporary; see main testenv.
-setenv =
-    ZOPE_INTERFACE_STRICT_IRO=0
+    ZODB
 
 [testenv:docs]
 basepython =
-    python3.7
+    python3
 commands =
     sphinx-build -b html -d docs/_build/doctrees docs docs/_build/html
     sphinx-build -b doctest -d docs/_build/doctrees docs docs/_build/doctest