Automated wheel creation with cmake and more

CMakeLists.txt

@@ -33,10 +33,6 @@ set(PROJECT_ISSUES  "https://github.com/IntelVCL/Open3D/issues")
 configure_file("${PROJECT_SOURCE_DIR}/src/Open3DConfig.h.in"
                "${PROJECT_SOURCE_DIR}/src/Open3DConfig.h")
 
-# configure a setup.py for the pip packaging
-configure_file("${PROJECT_SOURCE_DIR}/util/pip_package/setup.py.in"
-               "${PROJECT_SOURCE_DIR}/util/pip_package/setup.py")
-
 if(WIN32 AND NOT CYGWIN)
     set(DEF_INSTALL_CMAKE_DIR CMake)
 else()

src/Python/CMakeLists.txt

@@ -1,5 +1,15 @@
-# Require Python.
-find_package(PythonInterp REQUIRED)
+# Option 1: Do not define "PYTHON_EXECUTABLE", but run `cmake ..` within your
+#           virtual environment. CMake will pick up the python executable in the
+#           virtual environment.
+# Option 2: You can also define `cmake -DPYTHON_EXECUTABLE` to specify a python
+#           executable.
+if (NOT PYTHON_EXECUTABLE)
+    # find_program will returns the python executable in current PATH, which
+    # works with virtualenv
+    find_program(PYTHON_IN_PATH "python")
+    set(PYTHON_EXECUTABLE ${PYTHON_IN_PATH})
+endif()
+message("Using Python executable ${PYTHON_EXECUTABLE}")
 
 set(PACKAGE_NAME open3d)
 
@@ -19,13 +29,68 @@ pybind11_add_module(${PACKAGE_NAME}
 
 target_link_libraries(${PACKAGE_NAME} PRIVATE ${CMAKE_PROJECT_NAME})
 
+# At `make`: open3d.so (or the equivalents) will be created at
+# PYTHON_COMPILED_MODULE_DIR. The default locaiton is `build/lib/Python`
+set(PYTHON_COMPILED_MODULE_DIR "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/Python")
 set_target_properties(${PACKAGE_NAME} PROPERTIES
                       FOLDER "Python"
-                      LIBRARY_OUTPUT_DIRECTORY "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/Python"
-                      ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/Python")
+                      LIBRARY_OUTPUT_DIRECTORY "${PYTHON_COMPILED_MODULE_DIR}"
+                      ARCHIVE_OUTPUT_DIRECTORY "${PYTHON_COMPILED_MODULE_DIR}")
 
-# find the python site-packages location
-execute_process(COMMAND ${PYTHON_EXECUTABLE} -m site --user-site OUTPUT_VARIABLE PYTHON_SITE_PACKAGES OUTPUT_STRIP_TRAILING_WHITESPACE)
+# Use `make python-package` to create the python package in the build directory
+# The python package will be created at PYTHON_PACKAGE_DIR. It contains:
+# 1) Pure-python code and misc files, copied from src/Python/Package
+# 2) The compiled python-C++ module, i.e. open3d.so (or the equivalents)
+# 3) Configured files and supporting files
+# Note: `make python-package` clears PYTHON_COMPILED_MODULE_DIR first every time
+set(PYTHON_PACKAGE_DST_DIR "${CMAKE_BINARY_DIR}/lib/python_package")
+add_custom_target(python-package
+    COMMAND ${CMAKE_COMMAND}
+            -DPYTHON_PACKAGE_SRC_DIR=${CMAKE_CURRENT_SOURCE_DIR}/Package
+            -DPYTHON_PACKAGE_DST_DIR=${PYTHON_PACKAGE_DST_DIR}
+            -DPYTHON_COMPILED_MODULE_PATH=$<TARGET_FILE:${PACKAGE_NAME}>
+            -DPROJECT_EMAIL=${PROJECT_EMAIL}
+            -DPROJECT_HOME=${PROJECT_HOME}
+            -DPROJECT_DOCS=${PROJECT_DOCS}
+            -DPROJECT_CODE=${PROJECT_CODE}
+            -DPROJECT_ISSUES=${PROJECT_ISSUES}
+            -DPROJECT_VERSION=${PROJECT_VERSION}
+            -DPYPI_PACKAGE_NAME=open3d  # `open3d` is used in building from source
+            -P ${CMAKE_CURRENT_SOURCE_DIR}/make_python_package.cmake
+)
+
+# Use `make pip-wheel` to create the pip package in the build directory
+add_custom_target(pip-wheel
+    COMMAND ${PYTHON_EXECUTABLE} "setup.py" "bdist_wheel"
+    WORKING_DIRECTORY ${PYTHON_PACKAGE_DST_DIR}
+    DEPENDS python-package
+)
 
-# configure installer to copy the python module to the python site-packages
-install(TARGETS ${PACKAGE_NAME} DESTINATION ${PYTHON_SITE_PACKAGES})
+# Use `make install-pip-wheel` to install pip wheel package to the current
+# python environment.
+add_custom_target(install-pip-wheel
+    COMMAND ${CMAKE_COMMAND}
+            -DPYTHON_PACKAGE_DST_DIR=${PYTHON_PACKAGE_DST_DIR}
+            -P ${CMAKE_CURRENT_SOURCE_DIR}/install_pip_wheel.cmake
+    DEPENDS pip-wheel
+)
+
+################################################################################
+# Warning: Internal use only, consider droping this in the future
+# Use `make all-pip-wheels` to create the pip package in the build directory
+# This creates: open3d-python, py3d, open3d-original, open3d-official, open-3d
+#               pip wheels
+add_custom_target(all-pip-wheels
+    COMMAND ${CMAKE_COMMAND}
+        -DPYTHON_PACKAGE_SRC_DIR=${CMAKE_CURRENT_SOURCE_DIR}/Package
+        -DPYTHON_PACKAGE_DST_DIR=${PYTHON_PACKAGE_DST_DIR}
+        -DPYTHON_COMPILED_MODULE_PATH=$<TARGET_FILE:${PACKAGE_NAME}>
+        -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}
+        -DPROJECT_EMAIL=${PROJECT_EMAIL}
+        -DPROJECT_HOME=${PROJECT_HOME}
+        -DPROJECT_DOCS=${PROJECT_DOCS}
+        -DPROJECT_CODE=${PROJECT_CODE}
+        -DPROJECT_ISSUES=${PROJECT_ISSUES}
+        -DPROJECT_VERSION=${PROJECT_VERSION}
+        -P ${CMAKE_CURRENT_SOURCE_DIR}/make_all_pip_wheels.cmake
+)

src/Python/Package/MANIFEST.in

@@ -0,0 +1,31 @@
+# ----------------------------------------------------------------------------
+# -                        Open3D: www.open3d.org                            -
+# ----------------------------------------------------------------------------
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 www.open3d.org
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+# ----------------------------------------------------------------------------
+
+include README.rst
+include LICENSE.txt
+
+include open3d/open3d*.pyd
+include open3d/open3d*.so

src/Python/Package/open3d/__init__.py.in

@@ -0,0 +1,31 @@
+# ----------------------------------------------------------------------------
+# -                        Open3D: www.open3d.org                            -
+# ----------------------------------------------------------------------------
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 www.open3d.org
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+# ----------------------------------------------------------------------------
+
+import importlib
+
+globals().update(importlib.import_module('open3d.open3d').__dict__)
+
+__version__ = '@PROJECT_VERSION@'

src/Python/Package/setup.py.in

@@ -0,0 +1,86 @@
+# ----------------------------------------------------------------------------
+# -                        Open3D: www.open3d.org                            -
+# ----------------------------------------------------------------------------
+# The MIT License (MIT)
+#
+# Copyright (c) 2018 www.open3d.org
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+# ----------------------------------------------------------------------------
+
+from setuptools import setup, find_packages
+
+setup(
+    author='Open3D Team',
+    author_email='@PROJECT_EMAIL@',
+    classifiers=[
+        # https://pypi.org/pypi?%3Aaction=list_classifiers
+        "Development Status :: 3 - Alpha",
+        "Environment :: MacOS X",
+        "Environment :: Win32 (MS Windows)",
+        "Environment :: X11 Applications",
+        "Intended Audience :: Developers",
+        "Intended Audience :: Education",
+        "Intended Audience :: Other Audience",
+        "Intended Audience :: Science/Research",
+        "License :: OSI Approved :: MIT License",
+        "Natural Language :: English",
+        "Operating System :: POSIX :: Linux",
+        "Programming Language :: C",
+        "Programming Language :: C++",
+        "Programming Language :: Python :: 2",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
+        "Topic :: Education",
+        "Topic :: Multimedia :: Graphics :: 3D Modeling",
+        "Topic :: Multimedia :: Graphics :: 3D Rendering",
+        "Topic :: Multimedia :: Graphics :: Capture",
+        "Topic :: Multimedia :: Graphics :: Graphics Conversion",
+        "Topic :: Multimedia :: Graphics :: Viewers",
+        "Topic :: Scientific/Engineering",
+        "Topic :: Scientific/Engineering :: Mathematics",
+        "Topic :: Scientific/Engineering :: Visualization",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+        "Topic :: Utilities",
+    ],
+    description=[
+        "Open3D is an open-source library that supports rapid development of software that deals with 3D data."
+    ],
+    install_requires=[
+        'numpy',
+    ],
+    include_package_data=True,
+    keywords="3D reconstruction point cloud mesh RGB-D visualization",
+    license="MIT",
+    long_description=open('README.rst').read(),
+    # Name of the package on PyPI
+    name="@PYPI_PACKAGE_NAME@",
+    packages=[
+        'open3d',
+    ],
+    url="@PROJECT_HOME@",
+    project_urls={
+        'Documentation': '@PROJECT_DOCS@',
+        'Source code': '@PROJECT_CODE@',
+        'Issues': '@PROJECT_ISSUES@',
+    },
+    version='@PROJECT_VERSION@',
+)

src/Python/install_pip_wheel.cmake

@@ -0,0 +1,7 @@
+# We need this file for cross-platform support on Windows
+# For Ubuntu/Mac, we can simply do `pip install ${PYTHON_PACKAGE_DST_DIR}/dist/*.whl -U`
+
+# Note: Since `make python-package` clears PYTHON_COMPILED_MODULE_DIR every time,
+#       it is guaranteed that there is only one wheel in ${PYTHON_PACKAGE_DST_DIR}/dist/*.whl
+file(GLOB WHEEL_FILE "${PYTHON_PACKAGE_DST_DIR}/dist/*.whl")
+execute_process(COMMAND pip install ${WHEEL_FILE} -U)

src/Python/make_all_pip_wheels.cmake

@@ -0,0 +1,43 @@
+# Warning: Internal use only, consider droping this in the future
+# Use `make all-pip-wheels` to create the pip package in the build directory
+# This creates: open3d-python, py3d, open3d-original, open3d-official, open-3d
+#               pip wheels
+
+# Clean up directory
+file(REMOVE_RECURSE ${PYTHON_PACKAGE_DST_DIR})
+file(MAKE_DIRECTORY ${PYTHON_PACKAGE_DST_DIR}/open3d)
+
+# 1) Pure-python code and misc files, copied from src/Python/Package
+file(COPY ${PYTHON_PACKAGE_SRC_DIR}/
+     DESTINATION ${PYTHON_PACKAGE_DST_DIR}
+     PATTERN "*.in" EXCLUDE
+)
+
+# 2) The compiled python-C++ module, i.e. open3d.so (or the equivalents)
+get_filename_component(PYTHON_COMPILED_MODULE_NAME ${PYTHON_COMPILED_MODULE_PATH} NAME)
+file(COPY ${PYTHON_COMPILED_MODULE_PATH}
+     DESTINATION ${PYTHON_PACKAGE_DST_DIR}/open3d)
+
+# 3) Configured files and supporting files
+configure_file("${PYTHON_PACKAGE_SRC_DIR}/MANIFEST.in"
+               "${PYTHON_PACKAGE_DST_DIR}/MANIFEST.in" COPYONLY)
+configure_file("${PYTHON_PACKAGE_SRC_DIR}/open3d/__init__.py.in"
+               "${PYTHON_PACKAGE_DST_DIR}/open3d/__init__.py")
+
+# We distributes multiple PyPI pacakges under different names
+set(PYPI_PACKAGE_NAMES
+    open3d-python
+    py3d
+    open3d-original
+    open3d-official
+    open-3d
+)
+foreach (PYPI_PACKAGE_NAME ${PYPI_PACKAGE_NAMES})
+    message("Making PyPI package: ${PYPI_PACKAGE_NAME}...")
+    configure_file("${PYTHON_PACKAGE_SRC_DIR}/setup.py.in"
+                   "${PYTHON_PACKAGE_DST_DIR}/setup.py")
+    execute_process(
+        COMMAND ${PYTHON_EXECUTABLE} "setup.py" "bdist_wheel"
+        WORKING_DIRECTORY ${PYTHON_PACKAGE_DST_DIR}
+)
+endforeach()

src/Python/make_python_package.cmake

@@ -0,0 +1,27 @@
+# Create python pacakge. It contains
+# 1) Pure-python code and misc files, copied from src/Python/Package
+# 2) The compiled python-C++ module, i.e. open3d.so (or the equivalents)
+# 3) Configured files and supporting files
+
+# Clean up directory
+file(REMOVE_RECURSE ${PYTHON_PACKAGE_DST_DIR})
+file(MAKE_DIRECTORY ${PYTHON_PACKAGE_DST_DIR}/open3d)
+
+# 1) Pure-python code and misc files, copied from src/Python/Package
+file(COPY ${PYTHON_PACKAGE_SRC_DIR}/
+     DESTINATION ${PYTHON_PACKAGE_DST_DIR}
+     PATTERN "*.in" EXCLUDE
+)
+
+# 2) The compiled python-C++ module, i.e. open3d.so (or the equivalents)
+get_filename_component(PYTHON_COMPILED_MODULE_NAME ${PYTHON_COMPILED_MODULE_PATH} NAME)
+file(COPY ${PYTHON_COMPILED_MODULE_PATH}
+     DESTINATION ${PYTHON_PACKAGE_DST_DIR}/open3d)
+
+# 3) Configured files and supporting files
+configure_file("${PYTHON_PACKAGE_SRC_DIR}/setup.py.in"
+               "${PYTHON_PACKAGE_DST_DIR}/setup.py")
+configure_file("${PYTHON_PACKAGE_SRC_DIR}/MANIFEST.in"
+               "${PYTHON_PACKAGE_DST_DIR}/MANIFEST.in" COPYONLY)
+configure_file("${PYTHON_PACKAGE_SRC_DIR}/open3d/__init__.py.in"
+               "${PYTHON_PACKAGE_DST_DIR}/open3d/__init__.py")