setuptools_wrap: Add support for "cmake" and "hybrid" python package

An "hybrid" package is a python package that have some files living
in the project source tree and some files installed by CMake.

A "CMake" package is a python package that is fully generated and
installed by CMake without any of his files existing in the source
tree.

The proper functioning of skbuild depends on:
  value of 'cmake_source_dir'
  value of 'cmake_install_dir'
  layout of the source tree

A new test named "test_setup_inputs" has been added to test different
configurations.

skbuild/setuptools_wrap.py

@@ -8,12 +8,16 @@
 import argparse
 
 from contextlib import contextmanager
-from distutils.errors import DistutilsGetoptError, DistutilsArgError
+from distutils.errors import (DistutilsArgError,
+                              DistutilsError,
+                              DistutilsGetoptError)
+from shutil import copyfile
 
 from . import cmaker
 from .command import build, install, clean, bdist, bdist_wheel, egg_info, sdist
 from .constants import CMAKE_INSTALL_DIR
 from .exceptions import SKBuildError
+from .utils import (mkdir_p, PythonModuleFinder)
 
 # XXX If 'six' becomes a dependency, use 'six.StringIO' instead.
 try:
@@ -161,6 +165,52 @@ def _check_skbuild_parameters(skbuild_kw):
         ))
 
 
+def strip_package(package_parts, module_file):
+    """Given ``package_parts`` (e.g. ``['foo', 'bar']``) and a
+    ``module_file`` (e.g. ``foo/bar/jaz/rock/roll.py``), starting
+    from the left, this function will strip the parts of the path
+    matching the package parts and return a new string
+    (e.g ``jaz/rock/roll.py``).
+
+    The function will work as expected for either Windows or Unix-style
+    ``module_file`` and this independently of the platform.
+    """
+    if not package_parts or os.path.isabs(module_file):
+        return module_file
+
+    package = "/".join(package_parts)
+    module_dir = os.path.dirname(module_file.replace("\\", "/"))
+
+    module_dir = module_dir[:len(package)]
+
+    return module_file[len(package) + 1:] if module_dir.startswith(
+        package) else module_file
+
+
+def _package_data_contain_module(module, package_data):
+    """Return True if the ``module`` is contained
+    in the ``package_data``.
+
+    ``module`` is a tuple of the form
+    ``(package, modulename, module_file)``.
+    """
+    (package, _, module_file) = module
+    if package not in package_data:
+        return True
+    # We need to strip the package because a module entry
+    # usually looks like this:
+    #
+    #   ('foo.bar', 'module', 'foo/bar/module.py')
+    #
+    # and the entry in package_data would look like this:
+    #
+    #   {'foo.bar' : ['module.py']}
+    if (strip_package(package.split("."), module_file)
+            not in package_data[package]):
+        return True
+    return False
+
+
 def setup(*args, **kw):
     """This function wraps setup() so that we can run cmake, make,
     CMake build, then proceed as usual with setuptools, appending the
@@ -296,7 +346,64 @@ def setup(*args, **kw):
     _classify_files(cmkr.install(), package_data, package_prefixes,
                     py_modules, new_py_modules,
                     scripts, new_scripts,
-                    data_files)
+                    data_files,
+                    cmake_source_dir, skbuild_kw['cmake_install_dir'])
+
+    # Duplicate the dictionary to prevent its update while iterating
+    # over the modules.
+    cmake_package_data = dict(package_data)
+
+    try:
+        # Search for python modules in both the current directory
+        # and cmake install tree.
+        modules = PythonModuleFinder(
+            packages, package_dir, py_modules,
+            alternative_build_base=CMAKE_INSTALL_DIR
+        ).find_all_modules()
+    except DistutilsError as msg:
+        raise SystemExit("error: {}".format(str(msg)))
+
+    print("")
+
+    for entry in modules:
+
+        # Check if module file should be copied into the CMake install tree.
+        if not _package_data_contain_module(entry, cmake_package_data):
+            continue
+
+        (package, _, src_module_file) = entry
+
+        # Copy missing module file
+        dest_module_file = os.path.join(CMAKE_INSTALL_DIR, src_module_file)
+
+        # Create directory if needed
+        dest_module_dir = os.path.dirname(dest_module_file)
+        if not os.path.exists(dest_module_dir):
+            print("creating directory {}".format(dest_module_dir))
+            mkdir_p(dest_module_dir)
+
+        # Copy file
+        print("copying {} -> {}".format(src_module_file, dest_module_file))
+        copyfile(src_module_file, dest_module_file)
+
+        # Since the mapping in package_data expects the package to be associated
+        # with a list of files relative to the directory containing the package,
+        # the following section makes sure to strip the redundant part of the
+        # module file path.
+        # The redundant part should be stripped for both cmake_source_dir and
+        # the package.
+        package_parts = []
+        if cmake_source_dir:
+            package_parts = cmake_source_dir.split(os.path.sep)
+        package_parts += package.split(".")
+
+        stripped_module_file = strip_package(package_parts, src_module_file)
+
+        # Update list of files associated with the corresponding package
+        try:
+            package_data[package].append(stripped_module_file)
+        except KeyError:
+            package_data[package] = [stripped_module_file]
 
     kw['package_data'] = package_data
     kw['package_dir'] = {
@@ -328,6 +435,8 @@ def has_ext_modules(self):
             return True
     kw['distclass'] = BinaryDistribution
 
+    print("")
+
     return upstream_setup(*args, **kw)
 
 
@@ -376,7 +485,11 @@ def _collect_package_prefixes(package_dir, packages):
 def _classify_files(install_paths, package_data, package_prefixes,
                     py_modules, new_py_modules,
                     scripts, new_scripts,
-                    data_files):
+                    data_files,
+                    cmake_source_dir, cmake_install_dir):
+    assert not os.path.isabs(cmake_source_dir)
+    assert cmake_source_dir != "."
+
     install_root = os.path.join(os.getcwd(), CMAKE_INSTALL_DIR)
     for path in install_paths:
         found_package = False
@@ -395,6 +508,15 @@ def _classify_files(install_paths, package_data, package_prefixes,
         # peel off the 'skbuild' prefix
         path = os.path.relpath(path, CMAKE_INSTALL_DIR)
 
+        # If the CMake project lives in a sub-directory (e.g src), its
+        # include rules are relative to it. We need to prepend
+        # the source directory so that the remaining of the logic
+        # can successfully check if the path belongs to a package or
+        # if it is a module.
+        if (cmake_source_dir
+                and not path.startswith(cmake_source_dir)):
+            path = os.path.join(cmake_source_dir, path)
+
         # check to see if path is part of a package
         for prefix, package in package_prefixes:
             if path.startswith(prefix):