From dcbce167cf7bb210870cab24578315315441449b Mon Sep 17 00:00:00 2001 From: renaud gaudin Date: Fri, 5 Jun 2020 18:11:21 +0000 Subject: [PATCH] build and test on macOS - updated test's libzim version to 6.1.6 to match release - added a macos-latest OS entry - added separate macOS steps when required - using `macosx_10_9_x86-64` as platform name. There doesn't see to be a linuxmanyX equivalent. Not much of a problem now as 10.9 introduced 64b only. - release to PiPy using twine as the ga used was through docker which is linux-only on ga. - updated descriptions to inform user about macOS availability --- .github/workflows/release.yml | 68 +++++++++++++++++++++++++++-------- .github/workflows/test.yml | 48 ++++++++++++++++++------- README.md | 6 ++-- lib/README.md | 2 +- setup.py | 10 +++--- 5 files changed, 99 insertions(+), 35 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8a41c437..a3f51bbb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,16 +6,19 @@ on: - v* env: - LIBZIM_RELEASE: libzim_linux-x86_64-6.1.5 + LIBZIM_VERSION: 6.1.6 LIBZIM_INCLUDE_PATH: include/zim CYTHON_VERSION: 0.29.6 + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + # TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/ jobs: release: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest] + os: [ubuntu-latest, macos-latest] # TODO: expand this to cross-platform builds (see V2 approach below) # os: [ubuntu-latest, windows-latest, macos-latest] python-version: [3.6, 3.7, 3.8] @@ -29,11 +32,27 @@ jobs: python-version: ${{ matrix.python-version }} architecture: x64 + - name: set macOS environ + if: matrix.os == 'macos-latest' + run: | + echo ::set-env name=LIBZIM_EXT::dylib + echo ::set-env name=LIBZIM_RELEASE::libzim_macos-x86_64-$LIBZIM_VERSION + echo ::set-env name=LIBZIM_LIBRARY_PATH::lib/libzim.${LIBZIM_VERSION:0:1}.dylib + echo ::set-env name=PLAFTORM_NAME::macosx_10.9_x86_64 + + - name: set linux environ + if: matrix.os == 'ubuntu-latest' + run: | + echo ::set-env name=LIBZIM_EXT::so + echo ::set-env name=LIBZIM_RELEASE::libzim_linux-x86_64-$LIBZIM_VERSION + echo ::set-env name=LIBZIM_LIBRARY_PATH::lib/x86_64-linux-gnu/libzim.so.$LIBZIM_VERSION + echo ::set-env name=PLAFTORM_NAME::manylinux1_x86_64 + - name: Cache libzim dylib & headers uses: actions/cache@master id: cache-libzim with: - path: libzim_linux + path: libzim_dist key: ${{ env.LIBZIM_RELEASE }}-libzim-cache - name: Download libzim dylib & headers from OpenZIM.org releases @@ -41,13 +60,21 @@ jobs: run: | wget -q https://download.openzim.org/release/libzim/$LIBZIM_RELEASE.tar.gz tar --gunzip --extract --file=$LIBZIM_RELEASE.tar.gz - mv $LIBZIM_RELEASE libzim_linux + mv $LIBZIM_RELEASE libzim_dist - - name: Link libzim dylib & headers into workspace lib and include folders + - name: Link Linux libzim dylib & headers into workspace lib and include folders + if: matrix.os == 'ubuntu-latest' run: | - cp -dp $GITHUB_WORKSPACE/libzim_linux/lib/x86_64-linux-gnu/* lib/ - ln -s libzim.so.6 lib/libzim.so - ln -s $GITHUB_WORKSPACE/libzim_linux/$LIBZIM_INCLUDE_PATH include/zim + cp -dp $GITHUB_WORKSPACE/libzim_dist/lib/x86_64-linux-gnu/* lib/ + ln -s libzim.so.${LIBZIM_VERSION:0:1} lib/libzim.so + ln -s $GITHUB_WORKSPACE/libzim_dist/$LIBZIM_INCLUDE_PATH include/zim + + - name: Link macOS libzim dylib & headers into workspace lib and include folders + if: matrix.os == 'macos-latest' + run: | + cp -Pp $GITHUB_WORKSPACE/libzim_dist/lib/* lib/ + ln -sf libzim.${LIBZIM_VERSION:0:1}.dylib lib/libzim.dylib + ln -s $GITHUB_WORKSPACE/libzim_dist/$LIBZIM_INCLUDE_PATH include/zim - name: Build cython, sdist, and bdist_wheels run: | @@ -57,8 +84,17 @@ jobs: then python3 setup.py sdist fi - cp -p lib/libzim.so.6 libzim - python3 setup.py bdist_wheel --plat-name=manylinux1_x86_64 + + - name: add macOS libzim binary to source for wheel + if: matrix.os == 'macos-latest' + run: cp -p lib/libzim.${LIBZIM_VERSION:0:1}.dylib libzim + + - name: add Linux libzim binary to source for wheel + if: matrix.os == 'ubuntu-latest' + run: cp -p lib/libzim.so.${LIBZIM_VERSION:0:1} libzim + + - name: build wheel + run: python3 setup.py bdist_wheel --plat-name=$PLAFTORM_NAME - uses: actions/upload-artifact@v1 with: @@ -67,7 +103,11 @@ jobs: - name: Push release to PyPI if: github.event_name == 'release' - uses: pypa/gh-action-pypi-publish@master - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} + run: | + pip install --upgrade twine + twine check dist/* + if [ "${{ matrix.os }}" == "ubuntu-latest" -a "${{ matrix.python-version }}" = "3.6" ] + then + twine upload dist/*.tar.gz + fi + twine upload dist/*.whl diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a689dc47..d00346a5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,8 +2,7 @@ name: test on: [push] env: - LIBZIM_RELEASE: libzim_linux-x86_64-6.1.5 - LIBZIM_LIBRARY_PATH: lib/x86_64-linux-gnu/libzim.so.6.1.5 + LIBZIM_VERSION: 6.1.6 LIBZIM_INCLUDE_PATH: include/zim CYTHON_VERSION: 0.29.6 MAX_LINE_LENGTH: 110 @@ -37,7 +36,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest] + os: [ubuntu-latest, macos-latest] # TODO: expand this once macos and windows libzim releases become available # os: [ubuntu-latest, windows-latest, macos-latest] # alternatively we can compile libzim in docker and use the container as an action @@ -52,11 +51,25 @@ jobs: python-version: ${{ matrix.python }} architecture: x64 + - name: set macOS environ + if: matrix.os == 'macos-latest' + run: | + echo ::set-env name=LIBZIM_EXT::dylib + echo ::set-env name=LIBZIM_RELEASE::libzim_macos-x86_64-$LIBZIM_VERSION + echo ::set-env name=LIBZIM_LIBRARY_PATH::lib/libzim.${LIBZIM_VERSION:0:1}.dylib + + - name: set linux environ + if: matrix.os == 'ubuntu-latest' + run: | + echo ::set-env name=LIBZIM_EXT::so + echo ::set-env name=LIBZIM_RELEASE::libzim_linux-x86_64-$LIBZIM_VERSION + echo ::set-env name=LIBZIM_LIBRARY_PATH::lib/x86_64-linux-gnu/libzim.so.$LIBZIM_VERSION + - name: Cache libzim dylib & headers uses: actions/cache@master id: cache-libzim with: - path: libzim_linux + path: libzim_dist key: ${{ env.LIBZIM_RELEASE }}-libzim-cache - name: Download libzim dylib & headers from OpenZIM.org releases @@ -64,25 +77,34 @@ jobs: run: | wget -q https://download.openzim.org/release/libzim/$LIBZIM_RELEASE.tar.gz tar --gunzip --extract --file=$LIBZIM_RELEASE.tar.gz - mv $LIBZIM_RELEASE libzim_linux + mv $LIBZIM_RELEASE libzim_dist - name: Link libzim dylib & headers into workspace lib and include folders run: | - cp -p $GITHUB_WORKSPACE/libzim_linux/$LIBZIM_LIBRARY_PATH lib/libzim.so - cp -p $GITHUB_WORKSPACE/libzim_linux/$LIBZIM_LIBRARY_PATH lib/ - sudo ldconfig $GITHUB_WORKSPACE/lib - ln -s $GITHUB_WORKSPACE/libzim_linux/$LIBZIM_INCLUDE_PATH include/zim - + cp -p $GITHUB_WORKSPACE/libzim_dist/$LIBZIM_LIBRARY_PATH lib/libzim.$LIBZIM_EXT + cp -p $GITHUB_WORKSPACE/libzim_dist/$LIBZIM_LIBRARY_PATH lib/ + ln -s $GITHUB_WORKSPACE/libzim_dist/$LIBZIM_INCLUDE_PATH include/zim + export LD_LIBRARY_PATH="$PWD/lib:$LD_LIBRARY_PATH" + + - name: update Linux shared objects cache + if: matrix.os == 'ubuntu-latest' + run: | + sudo ldconfig $PWD/lib + + - name: update macOS shared objects cache + if: matrix.os == 'macos-latest' + run: | + export DYLD_LIBRARY_PATH="$PWD/lib:$DYLD_LIBRARY_PATH" + - name: Build cython, sdist, and bdist_wheel run: | pip install --upgrade cython==$CYTHON_VERSION setuptools pip wheel python3 setup.py build_ext python3 setup.py sdist bdist_wheel - + - name: Test built package with pytest run: | - export LD_LIBRARY_PATH=$PWD/lib:$LD_LIBRARY_PATH - sudo ldconfig pip install pytest pip install -e . + export DYLD_LIBRARY_PATH="$PWD/lib:$DYLD_LIBRARY_PATH" pytest . diff --git a/README.md b/README.md index e6508bdf..b9486bc5 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ See [example](examples/basic_writer.py) for a basic usage of the writer API. ## User Documentation -### Setup: Ubuntu/Debian `x86_64` (Recommended) +### Setup: Ubuntu/Debian and macOS `x86_64` (Recommended) Install the python `libzim` package from PyPI. @@ -50,12 +50,12 @@ Install the python `libzim` package from PyPI. pip3 install libzim ``` -The `x86_64` linux wheel automatically includes the `libzim.so` dylib and headers, but other platforms may need to install `libzim` and its headers manually. +The `x86_64` linux and macOS wheels automatically includes the `libzim.(so|dylib)` dylib and headers, but other platforms may need to install `libzim` and its headers manually. ### Installing the `libzim` dylib and headers manually -If you are not on a linux `x86_64` platform, you will have to install libzim manually. +If you are not on a linux or macOS `x86_64` platform, you will have to install libzim manually. Either by get a prebuilt binary at https://download.openzim.org/release/libzim or [compile `libzim` from source](https://github.com/openzim/libzim). diff --git a/lib/README.md b/lib/README.md index a33fa528..3d0c0063 100644 --- a/lib/README.md +++ b/lib/README.md @@ -1 +1 @@ -Put your libzim.so file in here. +Put your libzim.(so|dylib) file in here. diff --git a/setup.py b/setup.py index cf9ba845..24752411 100755 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ $ export LDFLAGS="-L/tmp/libzim_linux-x86_64-6.1.1/lib/x86_64-linux-gnu" $ export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/tmp/libzim_linux-x86_64-6.1.1/lib/x86_64-linux-gnu" """ +import platform from pathlib import Path from ctypes.util import find_library @@ -38,6 +39,7 @@ BASE_DIR = Path(__file__).parent LIBZIM_INCLUDE_DIR = 'include' # the libzim C++ header src dir (containing zim/*.h) LIBZIM_LIBRARY_DIR = 'lib' # the libzim .so binary lib dir (containing libzim.so) +LIBZIM_DYLIB = 'libzim.{ext}'.format(ext='dylib' if platform.system() == 'Darwin' else 'so') # Check for the CPP Libzim library headers in expected directory @@ -50,12 +52,12 @@ ) # Check for the CPP Libzim shared library in expected directory or system paths -if not ((BASE_DIR / LIBZIM_LIBRARY_DIR / 'libzim.so').exists() or find_library('zim')): +if not ((BASE_DIR / LIBZIM_LIBRARY_DIR / LIBZIM_DYLIB).exists() or find_library('zim')): print( - f"[!] Warning: Couldn't find libzim.so in ./{LIBZIM_LIBRARY_DIR} or system library paths!" + f"[!] Warning: Couldn't find {LIBZIM_DYLIB} in ./{LIBZIM_LIBRARY_DIR} or system library paths!" f" Hint: You can install it from source from https://github.com/openzim/libzim\n" - f" or download a prebuilt zimlib.so release into ./lib.\n" - f" (or set LDFLAGS='-L/lib/x86_64-linux-gnu')" + f" or download a prebuilt {LIBZIM_DYLIB} release into ./lib.\n" + f" (or set LDFLAGS='-L/lib/[x86_64-linux-gnu]')" ) def get_long_description():