ENH,WIP: Add support for including cython in coverage analysis

.coveragerc

@@ -1,3 +1,4 @@
 [run]
 omit=*/tests/*
 parallel=true
+plugins = Cython.Coverage

.travis.yml

@@ -81,7 +81,7 @@ before_install:
   - ccache -s
 
 install:
-  - pip install tox codecov
+  - pip install tox codecov Cython
   - ci/travis/get_hdf5_if_needed.sh
   - ls -lRa $HDF5_CACHE_DIR
 script:

appveyor.yml

@@ -43,7 +43,7 @@ install:
   - "py -3.6 -m pip install --upgrade wheel pip setuptools virtualenv"
   - "py -3.6 -m pip install requests"
   - "py -3.6 ci\\appveyor\\get_hdf5.py"
-  - "py -3.6 -m pip install tox codecov tox-appveyor"
+  - "py -3.6 -m pip install tox codecov Cython tox-appveyor"
 
 build: off
 

h5py/tests/conftest.py

@@ -1,3 +1,5 @@
+import os
+
 import h5py
 import pytest
 
@@ -6,3 +8,17 @@
 def writable_file(tmp_path):
     with h5py.File(tmp_path / 'test.h5', 'w') as f:
         yield f
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "subprocess(env=None): mark test to run in subprocess"
+    )
+
+
+def pytest_runtest_setup(item):
+    if "subprocess" in item.iter_markers():
+        if os.environ.get("CYTHON_COVERAGE"):
+            pytest.skip(
+                "subprocess tests cannot be run with CYTHON_COVERAGE enabled."
+            )

h5py/tests/test_filters.py

@@ -12,6 +12,7 @@
 
 """
 import numpy as np
+import pytest
 import h5py
 import pytest
 
@@ -80,6 +81,7 @@ def test_filter_ref_obj(writable_file):
 
 
 @pytest.mark.mpi_skip
+@pytest.mark.subprocess
 @insubprocess
 def test_unregister_filter(request):
     if h5py.h5z.filter_avail(h5py.h5z.FILTER_LZF):

h5py/tests/test_h5pl.py

@@ -23,6 +23,7 @@
 @pytest.mark.mpi_skip
 @insubprocess
 @subproc_env({'HDF5_PLUGIN_PATH': 'h5py_plugin_test'})
+@pytest.mark.subprocess(env={'HDF5_PLUGIN_PATH': 'h5py_plugin_test'})
 def test_default(request):
     assert h5pl.size() == 1
     assert h5pl.get(0) == b'h5py_plugin_test'
@@ -31,6 +32,7 @@ def test_default(request):
 @pytest.mark.mpi_skip
 @insubprocess
 @subproc_env({'HDF5_PLUGIN_PATH': 'h5py_plugin_test'})
+@pytest.mark.subprocess(env={'HDF5_PLUGIN_PATH': 'h5py_plugin_test'})
 def test_append(request):
     h5pl.append(b'/opt/hdf5/vendor-plugin')
     assert h5pl.size() == 2
@@ -41,6 +43,7 @@ def test_append(request):
 @pytest.mark.mpi_skip
 @insubprocess
 @subproc_env({'HDF5_PLUGIN_PATH': 'h5py_plugin_test'})
+@pytest.mark.subprocess(env={'HDF5_PLUGIN_PATH': 'h5py_plugin_test'})
 def test_prepend(request):
     h5pl.prepend(b'/opt/hdf5/vendor-plugin')
     assert h5pl.size() == 2
@@ -51,6 +54,7 @@ def test_prepend(request):
 @pytest.mark.mpi_skip
 @insubprocess
 @subproc_env({'HDF5_PLUGIN_PATH': 'h5py_plugin_test'})
+@pytest.mark.subprocess(env={'HDF5_PLUGIN_PATH': 'h5py_plugin_test'})
 def test_insert(request):
     h5pl.insert(b'/opt/hdf5/vendor-plugin', 0)
     assert h5pl.size() == 2
@@ -61,6 +65,7 @@ def test_insert(request):
 @pytest.mark.mpi_skip
 @insubprocess
 @subproc_env({'HDF5_PLUGIN_PATH': 'h5py_plugin_test'})
+@pytest.mark.subprocess(env={'HDF5_PLUGIN_PATH': 'h5py_plugin_test'})
 def test_replace(request):
     h5pl.replace(b'/opt/hdf5/vendor-plugin', 0)
     assert  h5pl.size() == 1
@@ -69,6 +74,7 @@ def test_replace(request):
 
 @pytest.mark.mpi_skip
 @insubprocess
+@pytest.mark.subprocess
 def test_remove(request):
     h5pl.remove(0)
     assert h5pl.size() == 0

setup.py

@@ -138,6 +138,9 @@ def run(self):
 if os.name == 'nt':
     package_data['h5py'].append('*.dll')
 
+#if os.environ.get('CYTHON_COVERAGE'):
+#    package_data['h5py'].extend(["*.pyx", "*.pxd"])
+
 setup(
   name = 'h5py',
   version = VERSION,

setup_build.py

@@ -43,13 +43,38 @@ def localpath(*args):
                       ]
 }
 
+CYTHON_SETTINGS = {
+    'compiler_directives': {}
+}
+
 if sys.platform.startswith('win'):
     COMPILER_SETTINGS['include_dirs'].append(localpath('windows'))
     COMPILER_SETTINGS['define_macros'].extend([
         ('_HDF5USEDLL_', None),
         ('H5_BUILT_AS_DYNAMIC_LIB', None)
     ])
 
+if os.environ.get('CYTHON_COVERAGE'):
+    CYTHON_SETTINGS['compiler_directives'].update(**{'linetrace': True})
+    COMPILER_SETTINGS['define_macros'].extend([
+        ('CYTHON_TRACE', '1'), ('CYTHON_TRACE_NOGIL', '1')
+    ])
+
+def cython_settings_to_args(cython_settings):
+    """
+    Converts cython compiler directives to cython cli arguments.
+
+    NOTE: ONLY SUPPORTS COMPILER DIRECTIVES, SUPPORT FOR ADDITIONAL KEYWORDS
+    DOES NOT EXIST AT THIS TIME
+    """
+    arguments = []
+    for setting_name, setting in cython_settings.items():
+        if setting_name != "compiler_directives":
+            raise Exception("Unable to produce matching cli argument to {}".format(setting_name))
+        for directive, dir_value in setting:
+            arguments.extend(['-X', directive + '=' + str(dir_value)])
+    return arguments
+
 
 class h5py_build_ext(build_ext):
 
@@ -102,7 +127,10 @@ def make_extension(module):
     @staticmethod
     def run_system_cython(pyx_files):
         try:
-            retcode = subprocess.call(['cython', '--fast-fail', '--verbose'] + pyx_files)
+            retcode = subprocess.call(
+                ['cython', '--fast-fail', '--verbose'] +
+                cython_settings_to_args(CYTHON_SETTINGS) + pyx_files
+            )
             if not retcode == 0:
                 raise Exception('ERROR: Cython failed')
         except OSError as e:
@@ -176,9 +204,12 @@ def run(self):
 
         # Run Cython
         print("Executing cythonize()")
-        self.extensions = cythonize(self._make_extensions(config),
-                                    force=config.rebuild_required or self.force,
-                                    language_level=3)
+        self.extensions = cythonize(
+            self._make_extensions(config),
+            force = config.rebuild_required or self.force,
+            language_level=3,
+            **CYTHON_SETTINGS
+        )
         self.check_rerun_cythonize()
 
         # Perform the build

tox.ini

@@ -49,6 +49,7 @@ whitelist_externals =
     mpirun
 setenv =
     COVERAGE_FILE={toxinidir}/.coverage_dir/coverage-{envname}
+    CYTHON_COVERAGE=true
 # needed otherwise coverage cannot find the file when reporting
 
 pip_pre =