Merge pull request #18 from svirpioj/master

Add Python packaging

.github/workflows/tests.yaml

@@ -0,0 +1,47 @@
+name: Run tests
+
+on: [push, pull_request]
+
+jobs:
+
+  build-and-test:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-20.04
+            boost_platform: 20.04
+            boost_version: 1.73.0
+          - os: macos-11
+            boost_platform: 11
+            boost_version: 1.73.0
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install boost
+        uses: MarkusJx/install-boost@v2.4.1
+        id: install-boost
+        with:
+            # REQUIRED: Specify the required boost version
+            # A list of supported versions can be found here:
+            # https://github.com/MarkusJx/prebuilt-boost/blob/main/versions-manifest.json
+            boost_version: ${{ matrix.boost_version }}
+            # OPTIONAL: Specify a platform version
+            platform_version: ${{ matrix.boost_platform }}
+      - name: Build
+        run: |
+          mkdir build
+          cd build
+          cmake .. -DCMAKE_BUILD_TYPE=Release -DENABLE_TESTING=1
+          cmake --build .
+        env:
+          BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }}
+      - name: Binary test
+        run: make test
+        working-directory: ./build/tests
+      - name: Python test
+        run: make test
+        working-directory: ./build/python-wrapper/tests

.github/workflows/wheels.yaml

@@ -0,0 +1,66 @@
+name: Build and publish wheels
+
+on: [push, pull_request]
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    # build on every tag starting with 'v'
+    # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04, macos-11]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      # Used to host cibuildwheel
+      - uses: actions/setup-python@v3
+
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel==2.11.3
+
+      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse
+        # to supply options, put them in 'env', like:
+        # env:
+        #   CIBW_SOME_OPTION: value
+
+      - uses: actions/upload-artifact@v3
+        with:
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - uses: actions/upload-artifact@v3
+        with:
+          path: dist/*.tar.gz
+
+  upload_pypi:
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    # upload to PyPI on every tag starting with 'v'
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    # alternatively, to publish when a GitHub Release is created, use the following rule:
+    # if: github.event_name == 'release' && github.event.action == 'published'
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          # unpacks default artifact into dist/
+          # if `name: artifact` is omitted, the action will create extra parent dir
+          name: artifact
+          path: dist
+
+      - uses: pypa/gh-action-pypi-publish@v1.5.0
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          # To test: repository_url: https://test.pypi.org/legacy/

CMakeLists.txt

@@ -1,6 +1,11 @@
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.12)
 PROJECT(varikn)
 
+SET(ENABLE_EXECUTABLES ON CACHE BOOL "Set false to skip building the executables")
+SET(ENABLE_TESTING OFF CACHE BOOL "Set true to enable testing")
+SET(ENABLE_PYTHON_WRAPPER ON CACHE BOOL "Set false to skip building the Python wrapper")
+SET(REQUIRE_PYTHON_WRAPPER OFF CACHE BOOL "Set true to fail if Python wrapper cannot be installed")
+
 set( CMAKE_CXX_STANDARD 11)
 
 cmake_policy(SET CMP0054 NEW)
@@ -49,12 +54,25 @@ ELSEIF(WIN32)
    #replace_flags(/MD /MT)
 ENDIF(APPLE)
 
-# setup custom output dirs
+# setup custom output dirs (unless defined)
 #
-set( EXECUTABLE_OUTPUT_PATH ${CMAKE_CURRENT_BINARY_DIR}/bin/${OUTPUT_DIR_NAME} )
-set( LIBRARY_OUTPUT_PATH    ${CMAKE_CURRENT_BINARY_DIR}/lib/${OUTPUT_DIR_NAME} )
-file(MAKE_DIRECTORY ${EXECUTABLE_OUTPUT_PATH})
-file(MAKE_DIRECTORY ${LIBRARY_OUTPUT_PATH})
+IF(NOT DEFINED CMAKE_RUNTIME_OUTPUT_DIRECTORY)
+  set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin)
+ENDIF()
+IF(NOT DEFINED CMAKE_LIBRARY_OUTPUT_DIRECTORY)
+  set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
+  set(PYTHON_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib/python/varikn)
+ELSE()
+  set(PYTHON_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
+ENDIF()
+IF(NOT DEFINED CMAKE_ARCHIVE_OUTPUT_DIRECTORY)
+  set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
+  set(PYTHON_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib/python/varikn)
+ELSE()
+  set(PYTHON_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY})
+ENDIF()
+file(MAKE_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
+file(MAKE_DIRECTORY ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 
 include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/src )
 file(GLOB c_srcs   RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "src/[^.]*.c")
@@ -75,13 +93,21 @@ install(TARGETS varikn  ARCHIVE DESTINATION lib/${OUTPUT_DIR_NAME})
 
 # other targets
 #
-add_subdirectory(app)
-add_subdirectory(python-wrapper)
-IF(CMAKE_ENABLE_TESTING)
+IF(ENABLE_EXECUTABLES)
+  add_subdirectory(app)
+ENDIF(ENABLE_EXECUTABLES)
+
+IF(ENABLE_PYTHON_WRAPPER)
+  add_subdirectory(python-wrapper)
+ENDIF(ENABLE_PYTHON_WRAPPER)
+
+IF(ENABLE_TESTING)
   enable_testing()
   add_subdirectory(tests)
-  add_subdirectory(python-wrapper/tests)
-ENDIF(CMAKE_ENABLE_TESTING)
+  IF(ENABLE_PYTHON_WRAPPER)
+    add_subdirectory(python-wrapper/tests)
+  ENDIF(ENABLE_PYTHON_WRAPPER)
+ENDIF(ENABLE_TESTING)
 
 # These are so that package name and install prefix can be overriden
 SET(DEB_INSTALL_PREFIX CACHE STRING "/usr")

install

@@ -65,15 +65,24 @@ Useful tricks:
 
 * Language wrappers DLLs can be built for Python using Swig.
   These are built automatically if the languages are detected by
-  cmake. You can explicitly enable/disable building for these
-  wrappers by passing -DPYTHON=1 (-DPYTHON=0). On Mac OS X, you can
+  cmake. You can explicitly enable or disable building for these
+  wrappers by passing -DENABLE_PYTHON_WRAPPER=1 or 0. On Mac OS X, you can
   force the python version to macports with -DMACPORTS_PYTHON_VERSION=2.6 .
   The default target is python3, see python_wrappers/Cmakelists.txt
   if you want to compile for python2.
 
+PYTHON INSTALL:
+
+The Python wrapper package (varikn) can be installed directly using pip:
+
+  pip install .
+
+Note that this installs only the Python wrapper, not the executables
+of the toolkit.
+
 TEST:
 
-Building Unit tests can be enabled with "-DCMAKE_ENABLE_TESTING=1". The tests
+Building Unit tests can be enabled with "-DENABLE_TESTING=1". The tests
 do not work on Windows yet. Unit tests can be run with "make test" or
 "ctest --verbose". Unit tests require the unit test library from Boost.
 
@@ -88,7 +97,7 @@ docker build -t varikn_test_container .
 # Log in container
 docker container run --interactive --tty  varikn_test_container
 # Inside container
-mkdir build; (cd build; cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_ENABLE_TESTING=1 && make && ctest --verbose)
+mkdir build; (cd build; cmake .. -DCMAKE_BUILD_TYPE=Release -DENABLE_TESTING=1 && make && ctest --verbose)
 
 # clean up
 docker image rm varikn_test_container

pyproject.toml

@@ -0,0 +1,7 @@
+[build-system]
+requires = [
+  "setuptools",
+  "setuptools_scm<7.0.0",  # For python3.6 support
+  "wheel"
+]
+build-backend = "setuptools.build_meta"

python-wrapper/CMakeLists.txt

@@ -1,19 +1,18 @@
-option(PYTHON2 "Lower the requirement for python to 2.7")
-if (PYTHON2)
-FIND_PACKAGE(PythonInterp 2.7)
-FIND_PACKAGE(PythonLibs 2.7)
-else()
-FIND_PACKAGE(PythonInterp 3.2)
-FIND_PACKAGE(PythonLibs 3.2)
-endif()
-# You should be able to compile python2 wrappers just by
-# decreasing the minimum required python version
+# manylinux dockers contain only Development.Module, but building on
+# macOS-11 fails without full Development
+IF (APPLE)
+  FIND_PACKAGE(Python 3.2 COMPONENTS Interpreter Development)
+ELSE()
+  FIND_PACKAGE(Python 3.2 COMPONENTS Interpreter Development.Module)
+ENDIF()
 
 FIND_PACKAGE(SWIG 2.0)
 
-if (PYTHONLIBS_FOUND AND SWIG_FOUND)
+if (Python_FOUND AND SWIG_FOUND)
     INCLUDE(${SWIG_USE_FILE})
-    set(LIBRARY_OUTPUT_PATH ${LIBRARY_OUTPUT_PATH}/python)
+    set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PYTHON_LIBRARY_OUTPUT_DIRECTORY})
+    set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${PYTHON_ARCHIVE_OUTPUT_DIRECTORY})
+    file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/varikn/__init__.py  DESTINATION ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 
     IF (APPLE AND MACPORTS_PYTHON_VERSION)
         #set( PLATFORM_LINKFLAGS "-framework Foundation")
@@ -41,34 +40,32 @@ if (PYTHONLIBS_FOUND AND SWIG_FOUND)
     ENDIF (APPLE AND MACPORTS_PYTHON_VERSION)
 
     ###########################
-    INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_PATH} ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/..)
+    INCLUDE_DIRECTORIES(${Python_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/..)
 
-   # swig settings
-   message("Python version ${PYTHON_VERSION}")
-   if(PYTHON_INCLUDE_PATH MATCHES "(python3)")
-      message("Using python ver >2, disabling FILE * typemaps")
-      SET(CMAKE_SWIG_FLAGS "-shadow" "-DPYTHON3" "-py3")
-    else( )
-      SET(CMAKE_SWIG_FLAGS "-shadow")
-    endif()
+    # swig settings
+    message("Python version ${Python_VERSION}")
+    SET(CMAKE_SWIG_FLAGS "-shadow" "-DPYTHON3" "-py3")
 
-    SET(CMAKE_SWIG_OUTDIR ${LIBRARY_OUTPUT_PATH})
+    SET(CMAKE_SWIG_OUTDIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
     SET_SOURCE_FILES_PROPERTIES(VariKN.i PROPERTIES CPLUSPLUS ON)
     # create lib
     swig_add_library(varikn_python LANGUAGE python SOURCES VariKN.i )
-    swig_link_libraries(varikn_python varikn ${PYTHON_LIBRARIES} ${PLATFORM_LINKFLAGS})
+    swig_link_libraries(varikn_python varikn ${Python_LIBRARIES} ${PLATFORM_LINKFLAGS})
     set_target_properties(${SWIG_MODULE_varikn_python_REAL_NAME} PROPERTIES OUTPUT_NAME "_varikn")
     install(TARGETS ${SWIG_MODULE_varikn_python_REAL_NAME} DESTINATION lib/site-packages)
-    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/../lib/python/varikn.py DESTINATION lib/site-packages)
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/../lib/python/varikn/wrapper.py DESTINATION lib/site-packages)
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/../lib/python/varikn/__init__.py DESTINATION lib/site-packages)
 
     message(STATUS "detecting Python")
-    message(STATUS "\t PYTHON_INCLUDE_PATH= ${PYTHON_INCLUDE_PATH}")
-    message(STATUS "\t PYTHON_LIBRARIES= ${PYTHON_LIBRARIES}")
-    message(STATUS "\t PYTHON_EXECUTABLE= ${PYTHON_EXECUTABLE}")
+    message(STATUS "\t Python_INCLUDE_DIRS= ${Python_INCLUDE_DIRS}")
+    message(STATUS "\t Python_LIBRARIES= ${Python_LIBRARIES}")
+    message(STATUS "\t Python_EXECUTABLE= ${Python_EXECUTABLE}")
     message(STATUS "\t Swig ${SWIG_VERSION}")
-    if (NOT PYTHONLIBS_FOUND)
-      message(FATAL_ERROR "Unable to find Python libs")
-    elseif (NOT SWIG_FOUND)
-      message(FATAL_ERROR "Unable to find Swig")
-    endif (NOT PYTHONLIBS_FOUND)
-endif(PYTHONLIBS_FOUND AND SWIG_FOUND)
+elseif(REQUIRE_PYTHON_WRAPPER)
+  if (NOT Python_FOUND)
+    message(FATAL_ERROR "Unable to find Python libs")
+  endif()
+  if (NOT SWIG_FOUND)
+    message(FATAL_ERROR "Unable to find Swig")
+  endif()
+endif()

python-wrapper/VariKN.i

@@ -1,7 +1,7 @@
 %include "exception.i"
 %include "std_string.i"
 %include "std_vector.i"
-%module varikn
+%module(package="varikn") wrapper
 
 %exception {
   try {
@@ -38,16 +38,6 @@
 %typemap(out) std::string& {
   $result = Py_BuildValue("s#",$1->c_str(),$1->size());
 }
-
-#ifndef PYTHON3
-## Python 3 no longer supports FILE* typemaps, this can be enabled for py2
-%typemap(in) FILE* {
-        if (!(PyFile_Check($input))) {
-                PyErr_SetString(PyExc_TypeError, "not a file pointer");
-                return NULL;
-        }
-        $1=PyFile_AsFile($input);}
-#endif
 #endif
 
 %inline %{

python-wrapper/tests/CMakeLists.txt

@@ -1,11 +1,13 @@
+FIND_PACKAGE(Python 3.2 COMPONENTS Interpreter)
+
 ADD_CUSTOM_COMMAND(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/UnitTests.py" 
   COMMAND cmake -E copy "${CMAKE_CURRENT_SOURCE_DIR}/UnitTests.py" 
   "${CMAKE_CURRENT_BINARY_DIR}" 
   DEPENDS   "${CMAKE_CURRENT_SOURCE_DIR}/UnitTests.py" 
   ) 
 
 FILE ( COPY "${CMAKE_CURRENT_SOURCE_DIR}/UnitTests.py" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}" )
-add_test( pytest "${PYTHON_EXECUTABLE}" ${CMAKE_CURRENT_BINARY_DIR}/UnitTests.py )
+add_test( pytest ${Python_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/UnitTests.py )
 
 # We should make sure that the c unit tests run before trying to run python unit tests.
 # However, the following does not work. FIXME !

python-wrapper/varikn/__init__.py

@@ -0,0 +1 @@
+from .wrapper import InterTreeGram, Perplexity, VarigramTrainer