CMakeLists.txt

project(PythonPackagesPlugin)

# Name and version of our package

set(PACKAGE_NAME PythonPackages)
set(PACKAGE_VERSION ${PYTHON_VERSION})

# Package repository and release tag

set(PACKAGE_REPOSITORY python)
set(RELEASE_TAG opencor_0.7)

# Git tags of Python package installers

set(PIP_GIT_TAG "19.2.3")
set(SETUPTOOLS_GIT_TAG "v41.2.0")

# The versions of the Python packages we bundle

# Science packages

set(NUMPY_VERSION 1.17.2)
set(SCIPY_VERSION 1.3.1)

set(MATPLOTLIB_VERSION 3.1.1)

# IPython, Jupyter, and friends

set(NOTEBOOK_VERSION 6.0.1)
set(IPYWIDGETS_VERSION 7.5.1)
set(NBCONVERT_VERSION 5.6.0)
set(JUPYTER_CONSOLE_VERSION 6.0.0)

set(JUPYTERLAB_VERSION 1.1.3)

# We use patched versions of these packages to support PythonQt

set(IPYKERNEL_VERSION 5.1.2)
set(IPYKERNEL_PYTHONQT 5.1.2.0)

set(IPYTHON_VERSION 7.8.0)
set(IPYTHON_PYTHONQT 7.8.0.0)

set(QTCONSOLE_VERSION 4.5.5)
set(QTCONSOLE_PYTHONQT 4.5.5.0)

set(TRAITLETS_VERSION 4.3.2)
set(TRAITLETS_PYTHONQT 4.3.2.1)

# Documentation packages

set(LXML_VERSION 4.4.1)
set(SPHINX_VERSION 2.2.0)

# Our GitHub organisation

set(GITHUB_OPENCOR https://github.com/opencor)

# Specify where our local package will be installed

string(REPLACE "${CMAKE_SOURCE_DIR}" "${CMAKE_SOURCE_DIR}/ext"
       FULL_LOCAL_EXTERNAL_PACKAGE_DIR "${PROJECT_SOURCE_DIR}/${EXTERNAL_PACKAGE_DIR}")

# Where Python packages are installed

set(LOCAL_SITE_PACKAGES_DIR ${PYTHON_RELATIVE_RUNTIME_DIR}/site-packages)
set(FULL_LOCAL_SITE_PACKAGES_DIR ${FULL_LOCAL_EXTERNAL_PACKAGE_DIR}/${LOCAL_SITE_PACKAGES_DIR})

# Directories we package

set(PACKAGED_FILES
    include
    ${LOCAL_SITE_PACKAGES_DIR}
    ${PYTHON_RELATIVE_SCRIPT_DIR}
)

# These directories don't exist for Windows debug mode

if(RELEASE_MODE OR NOT WIN32)
    list(APPEND PACKAGED_FILES
        etc
        share
    )
endif()

# Specify the files that need to have their SHA-1 value checked
# Note: at least one file must have its SHA-1 value checked...

if(WIN32)
    if(RELEASE_MODE)
        set(SHA1_FILES ${LOCAL_SITE_PACKAGES_DIR}/numpy/core/_multiarray_umath.cp${PYTHON_ABI_VERSION}-win_amd64.pyd)
    else()
        set(SHA1_FILES ${LOCAL_SITE_PACKAGES_DIR}/numpy/core/_multiarray_umath_d.cp${PYTHON_ABI_VERSION}-win_amd64.pyd)
    endif()
elseif(APPLE)
    set(SHA1_FILES ${LOCAL_SITE_PACKAGES_DIR}/numpy/core/_multiarray_umath.cpython-${PYTHON_ABI_VERSION}-darwin.so)
elseif(UNIX)
    set(SHA1_FILES ${LOCAL_SITE_PACKAGES_DIR}/numpy/core/_multiarray_umath.cpython-${PYTHON_ABI_VERSION}-x86_64-linux-gnu.so)
endif()

# Use the pre-built version of our package unless instructed otherwise

if(USE_PREBUILT_PYTHON_PACKAGES_PACKAGE)
    # Retrieve the plugin's package

    if(WIN32)
        if(RELEASE_MODE)
            retrieve_package_file(${PACKAGE_NAME} ${PACKAGE_VERSION}
                                  ${FULL_LOCAL_EXTERNAL_PACKAGE_DIR} 85ec07562073aab48c368de525dbf4399225c932
                                  PACKAGE_REPOSITORY ${PACKAGE_REPOSITORY}
                                  RELEASE_TAG ${RELEASE_TAG}
                                  SHA1_FILES ${SHA1_FILES}
                                  SHA1_VALUES 360dfb74ccaca4d0152f5184c9adfeb4a65fd5d3)
        else()
            retrieve_package_file(${PACKAGE_NAME} ${PACKAGE_VERSION}
                                  ${FULL_LOCAL_EXTERNAL_PACKAGE_DIR} 077f8f0f162fe54d1d4743b4ad43869d3683b8c7
                                  PACKAGE_REPOSITORY ${PACKAGE_REPOSITORY}
                                  RELEASE_TAG ${RELEASE_TAG}
                                  SHA1_FILES ${SHA1_FILES}
                                  SHA1_VALUES 3483da0d92f6b154ff7ce699350e741da0a84104)
        endif()
    elseif(APPLE)
        retrieve_package_file(${PACKAGE_NAME} ${PACKAGE_VERSION}
                              ${FULL_LOCAL_EXTERNAL_PACKAGE_DIR} 991ae47419a755a35be130584e4c1e3bab51f3fb
                              PACKAGE_REPOSITORY ${PACKAGE_REPOSITORY}
                              RELEASE_TAG ${RELEASE_TAG}
                              SHA1_FILES ${SHA1_FILES}
                              SHA1_VALUES 4443d8cb01dcc0edc148f9d095201b13fd09f923)
    else()
        retrieve_package_file(${PACKAGE_NAME} ${PACKAGE_VERSION}
                              ${FULL_LOCAL_EXTERNAL_PACKAGE_DIR} 23c1abf6b8b92b1e78c41b8d66bdcffd160af746
                              PACKAGE_REPOSITORY ${PACKAGE_REPOSITORY}
                              RELEASE_TAG ${RELEASE_TAG}
                              SHA1_FILES ${SHA1_FILES}
                              SHA1_VALUES 80632ca1842b5a8668616766ef3bb2d01ed84e63)
    endif()
else()
    # Install Python packages using pip

    set(PACKAGE_BUILD ${PACKAGE_NAME}Build)

    add_custom_target(${PACKAGE_BUILD})

    # Python packages installation requires Python

    set(BUILD_DEPENDENCIES PythonLibraryPlugin)

    # Clean our site packages directory before installing any packages

    file(REMOVE_RECURSE ${FULL_LOCAL_SITE_PACKAGES_DIR})

    file(MAKE_DIRECTORY ${FULL_LOCAL_SITE_PACKAGES_DIR})

    # Use our installed pip when upgrading pip

    file(TO_NATIVE_PATH ${FULL_LOCAL_EXTERNAL_PACKAGE_DIR} FULL_LOCAL_EXTERNAL_PACKAGE_PREFIX)

    set(PIP_INSTALL_COMMAND
        ${CMAKE_COMMAND} -E env "PYTHONPATH=${FULL_LOCAL_SITE_PACKAGES_DIR}"
        ${PYTHON_EXECUTABLE} -s -m
        pip install --prefix ${FULL_LOCAL_EXTERNAL_PACKAGE_PREFIX}
                    --upgrade --upgrade-strategy only-if-needed
                    --no-warn-script-location)

    # Install the Python package manager and its dependencies

    set(PIP_SOURCE_DIR ${CMAKE_SOURCE_DIR}/ext/pip)

    if(WIN32)
        # On Windows, we use our version of pip that is modified to create
        # *-script.py files when installing packages with console scripts, so
        # that they can then be updated to reference OpenCOR's Python executable

        # Install the Python setuptools package

        set(SETUPTOOLS_SOURCE_DIR ${CMAKE_SOURCE_DIR}/ext/setuptools)

        ExternalProject_Add(installSetuptools
            DOWNLOAD_DIR
                ${SETUPTOOLS_SOURCE_DIR}
            SOURCE_DIR
                ${SETUPTOOLS_SOURCE_DIR}
            BINARY_DIR
                ${SETUPTOOLS_SOURCE_DIR}
            INSTALL_DIR
                ${SETUPTOOLS_SOURCE_DIR}
            GIT_REPOSITORY
                https://github.com/pypa/setuptools.git
            GIT_TAG
                ${SETUPTOOLS_GIT_TAG}
            CONFIGURE_COMMAND
                ${PYTHON_EXECUTABLE} bootstrap.py
            BUILD_COMMAND
                ${PYTHON_EXECUTABLE} setup.py build
            INSTALL_COMMAND
                ${CMAKE_COMMAND} -E env "PYTHONPATH=${FULL_LOCAL_SITE_PACKAGES_DIR}"
                ${PYTHON_EXECUTABLE} setup.py install --prefix ${FULL_LOCAL_EXTERNAL_PACKAGE_PREFIX}
        )

        # Install our Python package installer

        ExternalProject_Add(installPip
            DOWNLOAD_DIR
                ${PIP_SOURCE_DIR}
            SOURCE_DIR
                ${PIP_SOURCE_DIR}
            BINARY_DIR
                ${PIP_SOURCE_DIR}
            INSTALL_DIR
                ${PIP_SOURCE_DIR}
            GIT_REPOSITORY
                https://github.com/opencor/pip.git
            GIT_TAG
                opencor_v${PIP_GIT_TAG}
            CONFIGURE_COMMAND
                ""
            BUILD_COMMAND
                ${CMAKE_COMMAND} -E env "PYTHONPATH=${FULL_LOCAL_SITE_PACKAGES_DIR}"
                ${PYTHON_EXECUTABLE} setup.py build
            INSTALL_COMMAND
                ${CMAKE_COMMAND} -E env "PYTHONPATH=${FULL_LOCAL_SITE_PACKAGES_DIR}"
                ${PYTHON_EXECUTABLE} setup.py install --prefix ${FULL_LOCAL_EXTERNAL_PACKAGE_PREFIX}
            DEPENDS
                installSetuptools
        )
    else()
        ExternalProject_Add(installPip
            DOWNLOAD_DIR
                ${PIP_SOURCE_DIR}
            SOURCE_DIR
                ${PIP_SOURCE_DIR}
            BINARY_DIR
                ${PIP_SOURCE_DIR}
            INSTALL_DIR
                ${PIP_SOURCE_DIR}
            URL
                https://bootstrap.pypa.io/get-pip.py
            DOWNLOAD_NO_EXTRACT
                TRUE
            CONFIGURE_COMMAND
                ""
            BUILD_COMMAND
                ""
            INSTALL_COMMAND
                ${CMAKE_COMMAND} -E env "PYTHONPATH=${FULL_LOCAL_SITE_PACKAGES_DIR}"
                ${PYTHON_EXECUTABLE} -s ${PIP_SOURCE_DIR}/get-pip.py
                    --prefix ${FULL_LOCAL_EXTERNAL_PACKAGE_PREFIX}
                    --no-wheel
        )

        # Make sure pip is up to date

        ExternalProject_Add_Step(installPip updatePip
                                 COMMAND ${PIP_INSTALL_COMMAND} pip
                                 WORKING_DIRECTORY ${FULL_DEST_EXTERNAL_BINARIES_DIR}
                                 DEPENDEES install)
    endif()

    add_dependencies(${PACKAGE_BUILD} installPip)
    add_dependencies(installPip ${BUILD_DEPENDENCIES})

    # We only install a minimal set of packages when building in Debug mode under Windows,
    # namely a minimal version of `numpy` (used to interfacing with our DataStore arrays)
    # and `sphinx` (used to build documentation)

    if (NOT RELEASE_MODE AND WIN32)
        # Download, build and install a debug version of numpy
        #
        # Note: This is a minimal build since we haven't provided linear algebra libraries
        #       nor a Fortran compiler

        set(NUMPY_SOURCE_DIR ${CMAKE_SOURCE_DIR}/ext/numpy)

        ExternalProject_Add(installNumpy
            DOWNLOAD_DIR
                ${NUMPY_SOURCE_DIR}
            SOURCE_DIR
                ${NUMPY_SOURCE_DIR}
            BINARY_DIR
                ${NUMPY_SOURCE_DIR}
            INSTALL_DIR
                ${NUMPY_SOURCE_DIR}
            URL
                https://github.com/numpy/numpy/releases/download/v${NUMPY_VERSION}/numpy-${NUMPY_VERSION}.zip
            CONFIGURE_COMMAND
                ""
            BUILD_COMMAND
                ${CMAKE_COMMAND} -E env "PYTHONPATH=${FULL_LOCAL_SITE_PACKAGES_DIR}"
                ${PYTHON_EXECUTABLE} setup.py build --debug
            INSTALL_COMMAND
                ${PIP_INSTALL_COMMAND}
                    --global-option=build
                    --global-option=--debug
                    .
        )

        add_dependencies(${PACKAGE_BUILD} installNumpy)
        add_dependencies(installNumpy installPip)
        add_dependencies(installNumpy ${BUILD_DEPENDENCIES})

        # Copy the numpy include directory to a place that OpenCOR can use

        add_custom_target(copyNumpyHeaders
            COMMAND ${CMAKE_COMMAND} -E copy_directory
                ${FULL_LOCAL_SITE_PACKAGES_DIR}/numpy/core/include include
            WORKING_DIRECTORY ${FULL_LOCAL_EXTERNAL_PACKAGE_DIR}
        )

        add_dependencies(${PACKAGE_BUILD} copyNumpyHeaders)
        add_dependencies(copyNumpyHeaders installNumpy)
        add_dependencies(copyNumpyHeaders ${BUILD_DEPENDENCIES})

        # Install Sphinx

        add_custom_target(installSphinx
            COMMAND ${PIP_INSTALL_COMMAND} --only-binary all
                sphinx==${SPHINX_VERSION}
            WORKING_DIRECTORY
                ${FULL_DEST_EXTERNAL_BINARIES_DIR}
            VERBATIM
        )

        add_dependencies(${PACKAGE_BUILD} installSphinx)
        add_dependencies(installSphinx installNumpy)
        add_dependencies(installSphinx ${BUILD_DEPENDENCIES})

    else()
        # Install numpy, scipy, and matplotlib

        add_custom_target(sciencePackages
            COMMAND ${PIP_INSTALL_COMMAND} --only-binary all
                numpy==${NUMPY_VERSION}
                scipy==${SCIPY_VERSION}
                matplotlib==${MATPLOTLIB_VERSION}
            WORKING_DIRECTORY
                ${FULL_DEST_EXTERNAL_BINARIES_DIR}
            VERBATIM
        )

        add_dependencies(${PACKAGE_BUILD} sciencePackages)
        add_dependencies(sciencePackages installPip)
        add_dependencies(sciencePackages ${BUILD_DEPENDENCIES})

        # Copy the numpy include directory to a place that OpenCOR can use

        add_custom_target(copyNumpyHeaders
            COMMAND ${CMAKE_COMMAND} -E copy_directory
                ${FULL_LOCAL_SITE_PACKAGES_DIR}/numpy/core/include include
            WORKING_DIRECTORY ${FULL_LOCAL_EXTERNAL_PACKAGE_DIR}
        )

        add_dependencies(${PACKAGE_BUILD} copyNumpyHeaders)
        add_dependencies(copyNumpyHeaders sciencePackages)
        add_dependencies(copyNumpyHeaders ${BUILD_DEPENDENCIES})

        # Install Jupyter, IPython and their dependencies
        # Note: order of installation matters as we want specific
        #       versions of each package

        add_custom_target(ipythonPackages
            COMMAND ${PIP_INSTALL_COMMAND} --only-binary all
                traitlets==${TRAITLETS_VERSION}
                ipython==${IPYTHON_VERSION}
                ipykernel==${IPYKERNEL_VERSION}
                qtconsole==${QTCONSOLE_VERSION}
                notebook==${NOTEBOOK_VERSION}
                jupyter_console==${JUPYTER_CONSOLE_VERSION}
                nbconvert==${NBCONVERT_VERSION}
                ipywidgets==${IPYWIDGETS_VERSION}
                jupyterlab==${JUPYTERLAB_VERSION}
            WORKING_DIRECTORY
                ${FULL_DEST_EXTERNAL_BINARIES_DIR}
            VERBATIM
        )

        add_dependencies(${PACKAGE_BUILD} ipythonPackages)
        add_dependencies(ipythonPackages sciencePackages)
        add_dependencies(ipythonPackages ${BUILD_DEPENDENCIES})

        # Install other packages we use

        add_custom_target(otherPackages
            COMMAND ${PIP_INSTALL_COMMAND} --only-binary all
                lxml==${LXML_VERSION}
            WORKING_DIRECTORY
                ${FULL_DEST_EXTERNAL_BINARIES_DIR}
            VERBATIM
        )

        add_dependencies(${PACKAGE_BUILD} otherPackages)
        add_dependencies(otherPackages ipythonPackages)
        add_dependencies(otherPackages ${BUILD_DEPENDENCIES})

        # Install Sphinx

        add_custom_target(installSphinx
            COMMAND ${PIP_INSTALL_COMMAND} --only-binary all
                sphinx==${SPHINX_VERSION}
            WORKING_DIRECTORY
                ${FULL_DEST_EXTERNAL_BINARIES_DIR}
            VERBATIM
        )

        add_dependencies(${PACKAGE_BUILD} installSphinx)
        add_dependencies(installSphinx otherPackages)
        add_dependencies(installSphinx ${BUILD_DEPENDENCIES})

        # We only upgrade to our versions of packages and patch matplotlib
        # *after* all 'standard' packages have been installed

        # Upgrade to our versions that have been patched to work with PythonQt

        set(ARCHIVE_PYTHONQT archive/PythonQt-v)

        add_custom_target(pythonQtUpdate
            COMMAND ${PIP_INSTALL_COMMAND} --no-deps --force-reinstall
                ${GITHUB_OPENCOR}/traitlets/${ARCHIVE_PYTHONQT}${TRAITLETS_PYTHONQT}.zip
                ${GITHUB_OPENCOR}/ipython/${ARCHIVE_PYTHONQT}${IPYTHON_PYTHONQT}.zip
                ${GITHUB_OPENCOR}/ipykernel/${ARCHIVE_PYTHONQT}${IPYKERNEL_PYTHONQT}.zip
                ${GITHUB_OPENCOR}/qtconsole/${ARCHIVE_PYTHONQT}${QTCONSOLE_PYTHONQT}.zip
            WORKING_DIRECTORY
                ${FULL_DEST_EXTERNAL_BINARIES_DIR}
            VERBATIM
        )

        add_dependencies(${PACKAGE_BUILD} pythonQtUpdate)
        add_dependencies(pythonQtUpdate installSphinx)
        add_dependencies(pythonQtUpdate ${BUILD_DEPENDENCIES})

        # Patch matplotlib's Qt backend so that it works with PythonQt

        add_custom_target(patchMatplotlib
            COMMAND ${PYTHON_EXECUTABLE} -s scripts/patch.py
                --strip=2
                --directory=${FULL_LOCAL_SITE_PACKAGES_DIR}/matplotlib
                --verbose
                src/matplotlib_${MATPLOTLIB_VERSION}.diff
            WORKING_DIRECTORY
                ${PROJECT_SOURCE_DIR}
            VERBATIM
        )

        add_dependencies(${PACKAGE_BUILD} patchMatplotlib)
        add_dependencies(patchMatplotlib pythonQtUpdate)
        add_dependencies(patchMatplotlib ${BUILD_DEPENDENCIES})
    endif()

    # Package the newly installed Python packages

    create_package_file(${PACKAGE_NAME} ${PACKAGE_VERSION}
                        PACKAGE_REPOSITORY ${PACKAGE_REPOSITORY}
                        RELEASE_TAG ${RELEASE_TAG}
                        PACKAGED_FILES ${PACKAGED_FILES}
                        SHA1_FILES ${SHA1_FILES}
                        TARGET ${PACKAGE_BUILD})

    set(DEPENDS_ON ${PACKAGE_BUILD})
endif()

# Allow other packages to use our scripts

if(WIN32)
    set(SPHINX_EXECUTABLE sphinx-build.exe)
else()
    set(SPHINX_EXECUTABLE sphinx-build)
endif()

set(SPHINX_EXECUTABLE
    ${CMAKE_COMMAND} -E env PYTHONPATH=${FULL_LOCAL_SITE_PACKAGES_DIR}
    ${FULL_LOCAL_EXTERNAL_PACKAGE_DIR}/${PYTHON_RELATIVE_SCRIPT_DIR}/${SPHINX_EXECUTABLE}
    PARENT_SCOPE
)

# Add the plugin

add_plugin(PythonPackages
    SOURCES
        ../../plugininfo.cpp

        src/pythonpackagesplugin.cpp
    PLUGINS
        PythonLibrary
    EXTERNAL_DESTINATION_DIR
        ${PYTHON_ROOT_DIR}
    EXTERNAL_SOURCE_DIR
        ${FULL_LOCAL_EXTERNAL_PACKAGE_DIR}
    DEPENDS_ON
        ${DEPENDS_ON}
)

add_dependencies(${PROJECT_BUILD_TARGET} ${PROJECT_NAME})

if(NOT "${DEPENDS_ON}" STREQUAL "")
    add_dependencies(${PROJECT_NAME} ${DEPENDS_ON})
endif()

# Update our list of Python specific dependencies

set(PYTHON_DEPENDENCIES ${PYTHON_DEPENDENCIES} ${INSTALL_EXTERNAL_FILES_TARGET} PARENT_SCOPE)

# Update the Python path in newly copied scripts

add_custom_command(TARGET ${PROJECT_NAME}
                   COMMAND ${PYTHON_EXECUTABLE} ${PYTHON_SCRIPT_DIR}/set_python_path.py
                                                --update-path ${FULL_LOCAL_EXTERNAL_PACKAGE_DIR}/${PYTHON_RELATIVE_SCRIPT_DIR}
                                                ${PYTHON_ROOT_DIR} -s)