Switch to cibuildwheel and build macOS wheels (#213)

* Switch to cibuildwheel and build macOS wheels

* Remove old scripts for building wheels

* Build Linux wheels on manylinux1 environment

.github/workflows/ci.yml

@@ -12,25 +12,44 @@ on:
 
 jobs:
   build-and-test-cpp:
-    runs-on: ubuntu-latest
+    name: Build and test C++ library on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
 
     steps:
       - uses: actions/checkout@v2
         with:
           submodules: recursive
 
-      - name: Build
+      - name: Build and install
+        if: startsWith(matrix.os, 'ubuntu')
         run: |
           cmake -DBUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=$PWD/install .
           make install
 
+      - name: Build and install
+        if: startsWith(matrix.os, 'macos')
+        run: |
+          cmake -DBUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=$PWD/install \
+            -DICU_INCLUDE_DIR=/usr/local/opt/icu4c/include \
+            -DICU_LIBRARY=/usr/local/opt/icu4c/lib/libicuuc.dylib .
+          make install
+
       - name: Test
         run: |
           test/onmt_tokenizer_test test/data
 
 
   build-and-test-python-wheels:
-    runs-on: ubuntu-latest
+    name: Build and test Python wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
 
     steps:
       - uses: actions/checkout@v2
@@ -44,21 +63,20 @@ jobs:
 
       - name: Build wheels
         run: |
-          docker run -v $PWD:/root -w /root quay.io/pypa/manylinux1_x86_64 bash /root/bindings/python/tools/build_wheel.sh
-
-      - name: Install dependencies
-        run: |
-          python -m pip install wheelhouse/*36*.whl
-          python -m pip install pytest
-
-      - name: Test with pytest
-        run: |
-          python -m pytest bindings/python/test/test.py
+          python -m pip install cibuildwheel==1.7.2
+          python -m cibuildwheel bindings/python --output-dir wheelhouse
+        env:
+          CIBW_BEFORE_ALL: bindings/python/tools/prepare_build_environment.sh
+          CIBW_BEFORE_BUILD: pip install pybind11==2.6.0
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux1
+          CIBW_TEST_COMMAND: pytest {project}/bindings/python/test/test.py
+          CIBW_TEST_REQUIRES: pytest
+          CIBW_SKIP: cp27-* pp* *i686
 
       - name: Upload Python wheels
         uses: actions/upload-artifact@v2
         with:
-          name: python-wheels-linux
+          name: python-wheels
           path: wheelhouse
 
 
@@ -71,7 +89,7 @@ jobs:
       - name: Download Python wheels
         uses: actions/download-artifact@v2
         with:
-          name: python-wheels-linux
+          name: python-wheels
 
       - name: Publish Python wheels to PyPI
         uses: pypa/gh-action-pypi-publish@master

bindings/python/Python.cc → bindings/python/pyonmttok/Python.cc

@@ -470,7 +470,7 @@ static ssize_t hash_token(const onmt::Token& token) {
                                  list_to_tuple(py::cast(token.features))));
 }
 
-PYBIND11_MODULE(pyonmttok, m)
+PYBIND11_MODULE(_ext, m)
 {
   m.def("is_placeholder", &onmt::Tokenizer::is_placeholder, py::arg("token"));
   m.def("set_random_seed", &onmt::set_random_seed, py::arg("seed"));

bindings/python/pyonmttok/__init__.py

@@ -0,0 +1 @@
+from ._ext import *

bindings/python/setup.py

@@ -1,7 +1,7 @@
 import os
 import sys
 
-from setuptools import setup, Extension
+from setuptools import find_packages, setup, Extension
 
 import pybind11
 
@@ -10,7 +10,7 @@
 library_dirs = []
 
 def _get_long_description():
-    readme_path = os.path.join(os.path.dirname(__file__), "..", "..", "README.md")
+    readme_path = "README.md"
     with open(readme_path) as readme_file:
         return readme_file.read()
 
@@ -27,13 +27,18 @@ def _maybe_add_library_root(lib_name, header_only=False):
 _maybe_add_library_root("TOKENIZER")
 
 cflags = ["-std=c++11", "-fvisibility=hidden"]
-if sys.platform == 'darwin':
-    cflags.append('-mmacosx-version-min=10.9')
+if sys.platform == "darwin":
+    cflags.append("-mmacosx-version-min=10.9")
+
+ldflags = []
+if sys.platform == "darwin":
+    ldflags.append("-Wl,-rpath,/usr/local/lib")
 
 tokenizer_module = Extension(
-    "pyonmttok",
-    sources=["Python.cc"],
+    "pyonmttok._ext",
+    sources=["pyonmttok/Python.cc"],
     extra_compile_args=cflags,
+    extra_link_args=ldflags,
     include_dirs=include_dirs,
     library_dirs=library_dirs,
     libraries=["OpenNMTTokenizer"])
@@ -67,6 +72,7 @@ def _maybe_add_library_root(lib_name, header_only=False):
         "Forum": "http://forum.opennmt.net/",
         "Source": "https://github.com/OpenNMT/Tokenizer/"
     },
+    packages=find_packages(),
     python_requires=">=3.5",
     setup_requires=["pytest-runner"],
     tests_require=["pytest"],

bindings/python/tools/prepare_build_environment.sh

@@ -0,0 +1,26 @@
+#! /bin/bash
+
+set -e
+set -x
+
+ROOT_DIR=$PWD
+ICU_VERSION=${ICU_VERSION:-64.2}
+
+# Install ICU.
+curl -L -O https://github.com/unicode-org/icu/releases/download/release-${ICU_VERSION/./-}/icu4c-${ICU_VERSION/./_}-src.tgz
+tar xf icu4c-*-src.tgz
+cd icu/source
+CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --disable-shared --enable-static
+make -j2 install
+cd $ROOT_DIR
+
+# Install cmake.
+pip install "cmake==3.18.*"
+
+# Build Tokenizer.
+rm -rf build
+mkdir build
+cd build
+cmake -DLIB_ONLY=ON ..
+make -j2 install
+cd $ROOT_DIR