diff --git a/.github/workflows/bak/cpythonlinuxIOnonregression.yml b/.github/workflows/bak/cpythonlinuxIOnonregression.yml index 7aa8cd88..47c9fdfb 100644 --- a/.github/workflows/bak/cpythonlinuxIOnonregression.yml +++ b/.github/workflows/bak/cpythonlinuxIOnonregression.yml @@ -21,7 +21,9 @@ jobs: python-version: [3.7] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 + with: + submodules: true - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: @@ -41,12 +43,10 @@ jobs: - name: Build gmic-cli if needed run: | sudo apt-get install -y libfftw3-dev libcurl4-openssl-dev libpng-dev zlib1g-dev libomp5 libomp-dev - bash build_tools.bash 1b_clean_and_regrab_gmic_src_and_make_cli if: steps.cache-gmic-cli.outputs.cache-hit != 'true' - name: Build gmic-py if needed run: | sudo apt-get install -y libfftw3-dev libcurl4-openssl-dev libpng-dev zlib1g-dev libomp5 libomp-dev - bash build_tools.bash 1_clean_and_regrab_gmic_src bash build_tools.bash 2_compile if: steps.cache-gmic-py.outputs.cache-hit != 'true' - name: run IO non regression tests using compiled gmic-py and gmic-cli diff --git a/.github/workflows/bak/cpythonmacosbuild_nosendpypi.yml b/.github/workflows/bak/cpythonmacosbuild_nosendpypi.yml index dfde44bc..130b0ff5 100644 --- a/.github/workflows/bak/cpythonmacosbuild_nosendpypi.yml +++ b/.github/workflows/bak/cpythonmacosbuild_nosendpypi.yml @@ -11,7 +11,10 @@ jobs: matrix: python-version: [3.7] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 + with: + submodules: true + # Prevent running this job on tag-based releases - name: Check if Git tag exists run: echo "HEAD_TAG=$(git tag --points-at HEAD)" >> $GITHUB_ENV @@ -46,7 +49,6 @@ jobs: # llvm 9 (latest) might cause issue with c++20 pre-support, see https://trac.macports.org/ticket/59575 and https://github.com/dtschump/gmic-py/commit/618d9d317d55aa7af3f85e92bfd07f06efea491f/checks?check_suite_id=368364476 # using llvm v6 instead brew install pkg-config fftw libpng libomp llvm@6 coreutils - bash build_tools.bash 1_clean_and_regrab_gmic_src MACOSX_DEPLOYMENT_TARGET=10.10 pip install git+https://github.com/mayeut/cibuildwheel.git MACOSX_DEPLOYMENT_TARGET=10.10 python -m cibuildwheel --output-dir wheelhouse echo "Mac OS wheelhouse after cibuildwheel looks like:" diff --git a/.github/workflows/bak/cpythonmacosbuildandreleaseontag.yml b/.github/workflows/bak/cpythonmacosbuildandreleaseontag.yml index 0f44cf02..02b13148 100644 --- a/.github/workflows/bak/cpythonmacosbuildandreleaseontag.yml +++ b/.github/workflows/bak/cpythonmacosbuildandreleaseontag.yml @@ -15,7 +15,10 @@ jobs: matrix: python-version: [3.7] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 + with: + submodules: true + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: @@ -42,7 +45,6 @@ jobs: # llvm 9 (latest) might cause issue with c++20 pre-support, see https://trac.macports.org/ticket/59575 and https://github.com/dtschump/gmic-py/commit/618d9d317d55aa7af3f85e92bfd07f06efea491f/checks?check_suite_id=368364476 # using llvm v6 instead brew install pkg-config fftw libpng libomp llvm@6 coreutils - bash build_tools.bash 1_clean_and_regrab_gmic_src MACOSX_DEPLOYMENT_TARGET=10.10 pip install git+https://github.com/mayeut/cibuildwheel.git MACOSX_DEPLOYMENT_TARGET=10.10 python -m cibuildwheel --output-dir wheelhouse && bash build_tools.bash 11_send_to_pypi echo "Mac OS wheelhouse after cibuildwheel and sending to pypi.org looks like:" diff --git a/.github/workflows/bak/cpythonmanylinuxbuild_nosendpypi.yml b/.github/workflows/bak/cpythonmanylinuxbuild_nosendpypi.yml index 90ed9f7a..5aaac4dd 100644 --- a/.github/workflows/bak/cpythonmanylinuxbuild_nosendpypi.yml +++ b/.github/workflows/bak/cpythonmanylinuxbuild_nosendpypi.yml @@ -21,7 +21,10 @@ jobs: docker_image: 'quay.io/pypa/manylinux2014_x86_64:latest' pre_cmd: steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 + with: + submodules: true + # Prevent running this job on tag-based releases - name: Check if Git tag exists run: echo "HEAD_TAG::$(git tag --points-at HEAD)" >> $GITHUB_ENV diff --git a/.github/workflows/bak/cpythonmanylinuxbuildandreleaseontag.yml b/.github/workflows/bak/cpythonmanylinuxbuildandreleaseontag.yml index 0f9d9993..9a3cd745 100644 --- a/.github/workflows/bak/cpythonmanylinuxbuildandreleaseontag.yml +++ b/.github/workflows/bak/cpythonmanylinuxbuildandreleaseontag.yml @@ -23,7 +23,10 @@ jobs: docker_image: 'quay.io/pypa/manylinux2014_x86_64:latest' pre_cmd: steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 + with: + submodules: true + # Detect G'MIC target version from VERSION file - name: Detect G'MIC target version run: echo "GHA_GMIC_VERSION=$(GHA_QUIET=1 bash build_tools.bash __get_py_package_version)" >> $GITHUB_ENV diff --git a/.github/workflows/bak/cpythonpythonpackageoptimized.yml b/.github/workflows/bak/cpythonpythonpackageoptimized.yml index f81ba471..03350448 100644 --- a/.github/workflows/bak/cpythonpythonpackageoptimized.yml +++ b/.github/workflows/bak/cpythonpythonpackageoptimized.yml @@ -43,7 +43,6 @@ jobs: bash build_tools.bash 21_check_c_style # should stop this script on style error bash build_tools.bash 23_check_python_style # should stop this script on style error - bash build_tools.bash 1_clean_and_regrab_gmic_src bash build_tools.bash 2_compile #2b_compile_debug bash build_tools.bash 3_test_compiled_so # && bash build_tools.bash 11_send_to_pypi #Note that most probably nothing will be sent to PyPI because of too modern linked libraries diff --git a/.github/workflows/bak/cpythonwindowsbuild.yml b/.github/workflows/bak/cpythonwindowsbuild.yml index 4237473d..a8484b89 100644 --- a/.github/workflows/bak/cpythonwindowsbuild.yml +++ b/.github/workflows/bak/cpythonwindowsbuild.yml @@ -7,31 +7,62 @@ on: [push] jobs: build-windows: runs-on: windows-latest - strategy: - matrix: - python-version: ['3.7'] + defaults: + run: + shell: msys2 {0} steps: - - uses: actions/checkout@v1 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + - uses: actions/checkout@v2 with: - python-version: ${{ matrix.python-version }} - - name: build on macos + submodules: true + + # Using https://github.com/marketplace/actions/setup-msys2 + - name: Using "Setup MSYS2" contrib + uses: msys2/setup-msys2@v2 + with: + update: true + install: make git + - run: git describe --dirty + - name: build on windows working-directory: ./ env: TWINE_PASSWORD_GITHUB_SECRET: ${{ secrets.TWINE_PASSWORD_GITHUB_SECRET }} # For build_tools.bash 11_send_to_pypi - PYTHON3: python - PIP3: pip - CIBW_BUILD: cp3*-*win* - # CIBW_ENVIRONMENT: "CC='/usr/local/opt/llvm@6/bin/clang' CXX='/usr/local/opt/llvm@6/bin/clang++' COMPILER_INDEX_STORE_ENABLE='NO'" - CIBW_BEFORE_BUILD: pip install -r dev-requirements.txt - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: python -W default -m pytest {project}/tests/test_gmic_py.py -vvv -rxXs - CIBW_BUILD_VERBOSITY: 3 run: | - # brew install pkg-config fftw libpng libomp llvm@6 - bash build_tools.bash 1_clean_and_regrab_gmic_src - pip install git+https://github.com/joerick/cibuildwheel.git - python -m cibuildwheel --output-dir wheelhouse - #echo "Mac OS wheelhouse after cibuildwheel looks like:" - #bash build_tools.bash 11_send_to_pypi + pacman --noconfirm -S unzip wget mingw-w64-x86_64-curl mingw-w64-x86_64-python-pip mingw-w64-x86_64-python python-devel mingw-w64-x86_64-gcc mingw-w64-x86_64-zlib mingw-w64-x86_64-libjpeg-turbo mingw64/mingw-w64-x86_64-pkgconf mingw-w64-x86_64-libpng mingw-w64-x86_64-libtiff mingw64/mingw-w64-x86_64-fftw + pacman --noconfirm -S mingw-w64-x86_64-python-numpy # used by unit tests, safer to install so than through pip on-the-fly source compilation + # Add complementary pkg-config .pc lookup path + export PKG_CONFIG_PATH=/usr/lib/pkgconfig/:$PKG_CONFIG_PATH + pkg-config --list-all + pkg-config libjpeg --libs --cflags + pkg-config libpng --libs --cflags + pkg-config libtiff-4 --libs --cflags + pkg-config zlib --libs --cflags + pkg-config libcurl --libs --cflags + pkg-config libcurl --libs --cflags + which python + python --version + gcc --version + g++ --version + which pip + pip --version + pip install -r dev-requirements.txt + bash build_tools.bash 2b_compile_debug + ldd $(find -name *.dll) + bash build_tools.bash 3_test_compiled_so format + bash build_tools.bash 4b_build_windows_portable_wheel + bash build_tools.bash 5_test_wheel format + bash build_tools.bash 5b_test_wheel_dlls_repaired + - name: upload .whl debug build dir as artifact + uses: actions/upload-artifact@v2 + with: + name: gmic-py-windows-debug-dist-dir + path: dist/ + - name: upload .whl repaired build dir as artifact + uses: actions/upload-artifact@v2 + with: + name: gmic-py-windows-repaired-dist-dir + path: wheelhouse/ + - name: upload .dll debug build dir as artifact + uses: actions/upload-artifact@v2 + with: + name: gmic-py-windows-debug-build-dir + path: build/ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..f419d706 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,100 @@ +# Automatically build binary wheels and source packages. +# Adapted from https://github.com/vinayak-mehta/pdftopng/blob/main/.github/workflows/build.yml +name: cibuildwheel + +on: [push] + +env: + CIBW_BUILD: "cp3?-manylinux_x86_64 cp3?-macosx_x86_64" + CIBW_SKIP: "cp35-*" + CIBW_BEFORE_BUILD_LINUX: "sh scripts/build_linux.sh" + CIBW_REPAIR_WHEEL_COMMAND_LINUX: "LD_LIBRARY_PATH=$(pwd)/lib/poppler/build/:$LD_LIBRARY_PATH auditwheel repair -w {dest_dir} {wheel}" + CIBW_BEFORE_BUILD_MACOS: "sh scripts/build_macos.sh" + CIBW_REPAIR_WHEEL_COMMAND_MACOS: "DYLD_LIBRARY_PATH=$(pwd)/lib/poppler/build:$DYLD_LIBRARY_PATH delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} -v {wheel}" + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "call scripts/wheel_repair.bat {wheel} {dest_dir}" + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest] + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: 3.8 + + - uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x86 + + - name: Build dependencies & wheels (Windows / x86) + if: runner.os == 'Windows' + shell: cmd + run: | + REM call scripts\build_win_x86.bat + cd gmic + git fetch --all --tags + git checkout v.2.9.1 + cd .. + C:\msys64\usr\bin\wget.exe -P gmic\src https://gmic.eu/gmic_stdlib.h + C:\msys64\usr\bin\wget.exe -P gmic\src https://github.com/dtschump/CImg/raw/v.2.9.1/CImg.h + vcpkg install libpng:x64-windows libjpeg-turbo:x64-windows zlib:x64-windows curl:x64-windows tiff:x64-windows fftw3:x64-windows + vcpkg list + python -m pip --disable-pip-version-check install cibuildwheel==1.6.1 wheel + python setup.py bdist_wheel + REM python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_BUILD: "cp3?-win32" + CIBW_SKIP: "cp35-*" + DISTUTILS_USE_SDK: 1 + MSSdk: 1 + +# - uses: ilammy/msvc-dev-cmd@v1 +# with: +# arch: amd64 +# +# - name: Build dependencies & wheels (Windows / amd64) +# if: runner.os == 'Windows' +# shell: cmd +# run: | +# call scripts\build_win_x64.bat +# python -m pip --disable-pip-version-check install cibuildwheel==1.6.1 +# python -m cibuildwheel --output-dir wheelhouse +# env: +# CIBW_BUILD: "cp3?-win_amd64" +# CIBW_SKIP: "cp35-*" +# DISTUTILS_USE_SDK: 1 +# MSSdk: 1 +# +# - name: Install cibuildwheel & build wheels (Linux & MacOS) +# if: runner.os != 'Windows' +# run: | +# python -m pip --disable-pip-version-check install cibuildwheel==1.6.1 +# python -m cibuildwheel --output-dir wheelhouse +# +# - uses: actions/upload-artifact@v2 +# with: +# path: ./wheelhouse/*.whl +# +# upload_pypi: +# needs: [build_wheels] +# runs-on: ubuntu-latest +# # upload to PyPI on every tag starting with 'v' +# if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') +# steps: +# - uses: actions/download-artifact@v2 +# with: +# name: artifact +# path: dist +# +# - uses: pypa/gh-action-pypi-publish@master +# with: +# user: __token__ +# password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/cpythonpythonpackage.yml b/.github/workflows/cpythonpythonpackage.yml index 23843f50..5cac85bb 100644 --- a/.github/workflows/cpythonpythonpackage.yml +++ b/.github/workflows/cpythonpythonpackage.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + # Prevent running this job on tag-based releases - name: Check if Git tag exists @@ -28,6 +28,10 @@ jobs: with: python-version: 3.7 + - uses: actions/checkout@v2 + with: + submodules: true + - name: Compile and run tests on .so, .whl and source distribution of gmic-py ${{ env.GHA_GMIC_VERSION }} working-directory: ./ env: @@ -35,13 +39,18 @@ jobs: PYTHON3: python3 PIP3: pip3 run: | + cd gmic + git fetch --all --tags + git checkout v.2.9.1 + cd .. for apt_file in `grep -lr microsoft /etc/apt/sources.list.d/`; do sudo rm $apt_file; done sudo apt-get update; sudo apt-get install -y libfftw3-dev libcurl4-openssl-dev libpng-dev zlib1g-dev libomp5 libomp-dev sudo apt-get install clang-format bash build_tools.bash 21_check_c_style # should stop this script on style error bash build_tools.bash 23_check_python_style # should stop this script on style error + wget -P gmic/src https://gmic.eu/gmic_stdlib.h + wget -P gmic/src https://github.com/dtschump/CImg/raw/v.2.9.1/CImg.h - bash build_tools.bash 1_clean_and_regrab_gmic_src bash build_tools.bash 2b_compile_debug ldd build/lib.linux-x86_64-3.*/gmic.cpython-3*-x86_64-linux-gnu.so # bash build_tools.bash 3_test_compiled_so # TODO tests exceptionnally disabled for remote testing diff --git a/.gitignore b/.gitignore index 66a01720..c1e4ef77 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ src/ out.json build/ dist/ -gmic docs/_build/ wheelhouse/* *.egg-info* diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..ac294e07 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "gmic"] + path = gmic + url = https://github.com/dtschump/gmic + branch = tags/2.9.1 diff --git a/build_tools.bash b/build_tools.bash index b1020acd..a1ec159c 100644 --- a/build_tools.bash +++ b/build_tools.bash @@ -5,6 +5,8 @@ PYTHON3=${PYTHON3:-python3} PIP3=${PIP3:-pip3} PYTHON_VERSION=$($PYTHON3 --version | cut -d' ' -f2 | cut -d'.' -f1,2) +WIN_DLL_DIR="win_dll" # Directory where gmic-py dependents .dll files will be stored + # Detect if Python is a debug build, per https://stackoverflow.com/a/647312/420684 # Useful for testing leaks within the compiled .so library if python -c "import sys; sys.exit(int(hasattr(sys, 'gettotalrefcount')))"; then @@ -47,7 +49,7 @@ function __get_py_package_version () { function 00_all_steps () { # See related but defunct Dockerfile at https://github.com/myselfhimself/gmic-py/blob/fc12cb74f4b02fbfd83e9e9fba44ba7a4cee0d93/Dockerfile - 21_check_c_style && 23_check_python_style && 1_clean_and_regrab_gmic_src && 2_compile && 3_test_compiled_so && 4_build_wheel && 5_test_wheel && 6_build_sdist && 7_test_sdist + 21_check_c_style && 23_check_python_style && 2_compile && 3_test_compiled_so && 4_build_wheel && 5_test_wheel && 6_build_sdist && 7_test_sdist echo "This is the final file tree:" find . } @@ -108,6 +110,7 @@ function 11_send_to_pypi () { set +x } +# Now replaced by git submodule, kept for legacy function 1_clean_and_regrab_gmic_src () { set -x GMIC_ARCHIVE_GLOB=gmic_${GMIC_VERSION}.tar.gz @@ -217,24 +220,40 @@ function 33_build_manylinux () { } function 3_test_compiled_so () { + set -x # Example usage: 3_test_compiled_so my_pytest_expr ====> pytest -k my_pytest_expr # Example usage: 3_test_compiled_so my_pytest_expr -x --pdb ====> pytest -k my_pytest_expr -x --pdb PYTEST_EXPRESSION_PARAM= if ! [ -z "$1" ]; then PYTEST_EXPRESSION_PARAM="-k ${@:1}" fi - if ! [ -z "$PYTHON_DEBUG" ]; then - GMIC_LIB_DIR="./build/lib*$PYTHON_VERSION*debug*/" + + # No python path must be set for the case 5_test_wheel + # Otherwise add the path of directory containing .so (debugging or optimized version) + if [ -z "$TEST_WHEEL" ]; then + pip uninstall gmic -y + if ! [ -z "$PYTHON_DEBUG" ]; then + export PYTHONPATH=$(readlink -f ./build/lib*$PYTHON_VERSION*debug*/) + else + export PYTHONPATH=$(readlink -f ./build/lib*$PYTHON_VERSION/) + fi else - GMIC_LIB_DIR="./build/lib*$PYTHON_VERSION/" + export PYTHONPATH="" fi - TEST_FILES="${TEST_FILES:-../../tests/test_gmic_py.py ../../tests/test_gmic_numpy.py ../../tests/test_gmic_numpy_toolkits.py ../../tests/test_gmic_py_memfreeing.py}" - #TEST_FILES="${TEST_FILES:-../../tests/test_gmic_py_memfreeing.py}" + $PYTHON3 -c "import gmic; print(gmic.__spec__)" + ls -l $PYTHONPATH + + TEST_FILES="${TEST_FILES:-tests/test_gmic_py.py tests/test_gmic_numpy.py tests/test_gmic_numpy_toolkits.py tests/test_gmic_py_memfreeing.py}" + + REQUIREMENTS="-r dev-requirements.txt -r test-requirements.txt" + if ! [ -z "$MSYSTEM" ]; then #windows / msys2 related + REQUIREMENTS="-r dev-requirements.txt -r test-requirements-win.txt" + fi + #TEST_FILES="${TEST_FILES:-tests/test_gmic_py_memfreeing.py}" FAILED_SUITES=0 - $PIP3 uninstall gmic -y; cd $GMIC_LIB_DIR ; LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH ; $PIP3 install -r ../../dev-requirements.txt ; pwd; ls; + $PIP3 install $REQUIREMENTS; pwd; ls; for TEST_FILE in $TEST_FILES; do - # $PIP3 uninstall gmic -y; cd $GMIC_LIB_DIR ; LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH ; $PIP3 install -r ../../dev-requirements.txt ; pwd; ls; PYTHONMALLOC=malloc valgrind --show-leak-kinds=all --leak-check=full --log-file=/tmp/valgrind-output $PYTHON3 -m pytest $TEST_FILES $PYTEST_EXPRESSION_PARAM -vvv -rxXs || { echo "Fatal error while running pytests" ; exit 1 ; } ; cd ../.. $PYTHON3 -m pytest $TEST_FILE $PYTEST_EXPRESSION_PARAM -vvv -rxX || { echo "Fatal error while running pytest suite $TEST_FILE" ; FAILED_SUITES=$((FAILED_SUITES+1)) ; } done cd ../.. @@ -244,10 +263,12 @@ function 3_test_compiled_so () { else return 0; fi + export PYTHONPATH="" + set +x } function 3b_test_compiled_so_no_numpy () { - TEST_FILES="../../tests/test_gmic_py.py" 3_test_compiled_so + TEST_FILES="tests/test_gmic_py.py" 3_test_compiled_so } function 31_test_compiled_so_filters_io () { @@ -269,20 +290,72 @@ function 31_test_compiled_so_filters_io () { PYTEST_NB_THREADS="-n 4" fi find ~/.config/gmic - $PIP3 uninstall gmic -y; cd ./build/lib*$PYTHON_VERSION*/ ; LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH ; $PIP3 install -r ../../dev-requirements.txt ; pwd; ls; $PYTHON3 -m pytest ../../tests/test_gmic_py_filters_io.py $PYTEST_EXPRESSION_PARAM $PYTEST_NB_THREADS -vvv -rxXs || { echo "Fatal error while running pytests" ; exit 1 ; } ; cd ../.. + $PIP3 uninstall gmic -y; cd ./build/lib*$PYTHON_VERSION*/ ; export PYTHONPATH=`pwd` ; $PIP3 install -r ../../{test,dev}-requirements.txt ; pwd; ls; $PYTHON3 -m pytest ../../tests/test_gmic_py_filters_io.py $PYTEST_EXPRESSION_PARAM $PYTEST_NB_THREADS -vvv -rxXs || { echo "Fatal error while running pytests" ; exit 1 ; } ; cd ../.. find ~/.config/gmic } function 4_build_wheel () { + rm -rf .egg-info # https://stackoverflow.com/questions/18085571/pip-install-error-setup-script-specifies-an-absolute-path#comment94579634_39198529 + $PIP3 install wheel || { echo "Fatal wheel package install error" ; exit 1 ; } $PYTHON3 setup.py bdist_wheel || { echo "Fatal wheel build error" ; exit 1 ; } - echo "Not doing any auditwheel repair step here, development environment :)" + + if [ -z "WHEEL_REPAIR" ]; then + echo "Not doing any auditwheel repair step here, development environment :)" + elif ! [ -z "$MSYSTEM" ]; then + echo "about to repair wheel for windows using wheel_repair.py" + echo ".dll ldd for info, before repair attempt:" + ldd build/lib.*/gmic-*.dll + #4c_copy_windows_dlls_for_repair + pip install -r dev-requirements-win.txt + LAST_WHEEL=`ls -Art dist/*.whl | tail -n 1` + $PYTHON3 wheel_repair.py $LAST_WHEEL -d "D:/a/_temp/msys/msys64/mingw64/bin/" + fi +} + +function 4b_build_windows_portable_wheel () { + WHEEL_REPAIR=1 4_build_wheel +} + + +function 4c_copy_windows_dlls_for_repair () { + set -x + # Inspired by gmic CLI Windows build instructions at https://gmic.eu/download.html#windows + mkdir -p $WIN_DLL_DIR + DLLS_TO_COPY=$(ldd build/lib.*/gmic-*.dll | grep -Eio "/mingw64/bin/.*.dll" | paste -s -d' ') + echo "DLLs to be copied into wheel are: $DLLS_TO_COPY" + cp $DLLS_TO_COPY $WIN_DLL_DIR || { echo "Could not copy dlls to $WIN_DLL_DIR" ; exit 1 ; } + find $WIN_DLL_DIR + set +x } function 5_test_wheel () { - $PIP3 install dist/gmic*.whl --no-cache-dir - $PYTHON3 -m pytest tests/test_gmic_py.py -rxXs -vvv + set -x + export PYTHONPATH="" + $PIP3 install `ls -Art dist/*.whl | tail -n 1` --no-cache-dir + TEST_WHEEL=1 3_test_compiled_so ${@:1} + $PIP3 uninstall gmic -y + set +x +} + +function 5b_test_wheel_dlls_repaired () { + set -x + LAST_WHEEL=`ls -Art wheelhouse/*.whl | tail -n 1` + unzip -l $LAST_WHEEL + PACKED_DLLS_FOUND=`unzip -l $LAST_WHEEL | grep -Eio "[a-z].*\.dll" | grep -v gmic-cpython | paste -s -d' '` + if [ -z "$PACKED_DLLS_FOUND" ]; then + echo "Error: non-gmic DLLs not found in wheel."; exit 1 + else + echo "Found non-gmic DLLs in wheel: $PACKED_DLLS_FOUND" + fi + + export PYTHONPATH="" + $PIP3 uninstall gmic -y + $PIP3 install $LAST_WHEEL --no-cache-dir + TEST_WHEEL=1 3_test_compiled_so ${@:1} $PIP3 uninstall gmic -y + + set +x } function 6_make_full_doc () { diff --git a/dev-requirements-win.txt b/dev-requirements-win.txt new file mode 100644 index 00000000..32f38cba --- /dev/null +++ b/dev-requirements-win.txt @@ -0,0 +1,7 @@ +# Requirements for a local developer environment and for CI building (shared library and wheel-level) +wheel +pkgconfig +pefile +machomachomangler + +# Do not put the 'black' Python code formatter as a dependency here it will not install for Python <= 3.6 and crash the Mac OS CI, black must be installed at formatting check time, outside any docker wheel-building environment. See the related formatting functions in build_tools.bash script. diff --git a/dev-requirements.txt b/dev-requirements.txt index c3cacd9c..008353e7 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,11 +1,5 @@ -# Requirements for a local developer environment and for CI building (top and wheel-level) +# Requirements for a local developer environment and for CI building (shared library and wheel-level) wheel pkgconfig -#scikit-image -numpy #==1.16.1 # fixes https://github.com/scikit-image/scikit-image/issues/3655 -Pillow -pytest -setuptools -pytest-xdist #allow parallelization -psutil + # Do not put the 'black' Python code formatter as a dependency here it will not install for Python <= 3.6 and crash the Mac OS CI, black must be installed at formatting check time, outside any docker wheel-building environment. See the related formatting functions in build_tools.bash script. diff --git a/gmic b/gmic new file mode 160000 index 00000000..0c98c226 --- /dev/null +++ b/gmic @@ -0,0 +1 @@ +Subproject commit 0c98c226a68bd582600eefe5eadd7118410d4c82 diff --git a/gmicpy.h b/gmicpy.h index 9f24713f..9c9d0eb1 100644 --- a/gmicpy.h +++ b/gmicpy.h @@ -1,4 +1,4 @@ -#include "gmic.h" +#include "gmic/src/gmic.h" // Set T be a float if not platform-overridden #ifndef T diff --git a/setup.cfg b/setup.cfg index ad150636..e4b0810b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,4 @@ [metadata] # This includes the license file(s) in the wheel. # https://wheel.readthedocs.io/en/stable/user_guide.html#including-license-files-in-the-generated-wheel-file -license_files = COPYING +license_file = COPYING diff --git a/setup.py b/setup.py index e513c448..162dc0c3 100644 --- a/setup.py +++ b/setup.py @@ -2,21 +2,30 @@ """ from os import path, listdir, environ +import os import sys -import platform from setuptools import setup, Extension, find_packages -import pkgconfig + +try: + import pkgconfig +except: + print("No pkgconfig found") + +IS_WINDOWS = sys.platform == "win32" here = path.abspath(path.dirname(__file__)) -gmic_src_path = path.abspath("src/gmic/src") +gmic_src_path = path.abspath("gmic/src") +WIN_DLL_DIR = path.abspath("win_dll") + +debug_enabled = "--debug" in sys.argv # List of non-standard '-l*' compiler parameters extra_link_args = [] -# List of libs to get include directories and linkable libraries paths from for compiling -pkgconfig_list = ["zlib"] +# Extra C flags +extra_compile_args = [] # Macros to toggle (gmic or CImg will do C/C++ #ifdef checks on them, testing mostly only their existence) # cimg_date and cimg_date are true variables, the value of which is checked in the C/C++ source @@ -34,68 +43,91 @@ ), # "display" statements display for Jupyter/Ipython ] -# Only require x11 if found -if pkgconfig.exists("x11"): - define_macros += [("cimg_display", None)] - pkgconfig_list += ["x11"] +# UNIX build flags pregeneration +if not IS_WINDOWS: -# Only require libpng if found -if pkgconfig.exists("libpng"): - define_macros += [("cimg_use_png", None)] - pkgconfig_list += ["libpng"] + extra_compile_args.extend(["-std=c++11"]) + if debug_enabled: + extra_compile_args += ["-O0"] + extra_compile_args += ["-g"] + else: + extra_compile_args += ["-Ofast", "-flto"] + extra_link_args += ["-flto"] -# Only require libtiff if found -if pkgconfig.exists("libtiff-4"): - define_macros += [("cimg_use_tiff", None)] - pkgconfig_list += ["libtiff-4"] + # List of libs to get include directories and linkable libraries paths from for compiling + pkgconfig_list = ["zlib"] -# Only require libjpeg if found -if pkgconfig.exists("libjpeg"): - define_macros += [("cimg_use_jpeg", None)] - pkgconfig_list += ["libjpeg"] + # Only require x11 if found + if pkgconfig.exists("x11"): + define_macros += [("cimg_display", None)] + pkgconfig_list += ["x11"] -# Only require fftw3 if found (non-2^ size image processing fails without it) -# We do not toggle cimg_use_fftw3, it is buggy -if pkgconfig.exists("fftw3"): - define_macros += [("cimg_use_fftw3", None)] - pkgconfig_list += ["fftw3"] - extra_link_args += ["-lfftw3_threads"] + # Only require libpng if found + if pkgconfig.exists("libpng"): + define_macros += [("cimg_use_png", None)] + pkgconfig_list += ["libpng"] -# Only compile with OpenCV if exists (nice for the 'camera' G'MIC command :-D ) -if pkgconfig.exists("opencv"): - define_macros += [("cimg_use_opencv", None)] - pkgconfig_list += ["opencv"] + # Only require libtiff if found + if pkgconfig.exists("libtiff-4"): + define_macros += [("cimg_use_tiff", None)] + pkgconfig_list += ["libtiff-4"] -if pkgconfig.exists("libcurl"): - define_macros += [("cimg_use_curl", None)] - pkgconfig_list += ["libcurl"] + # Only require libjpeg if found + if pkgconfig.exists("libjpeg"): + define_macros += [("cimg_use_jpeg", None)] + pkgconfig_list += ["libjpeg"] -packages = pkgconfig.parse(" ".join(pkgconfig_list)) -libraries = packages["libraries"] + [ - "pthread" -] # removed core-dumping 'gomp' temporarily (for manylinux builds) + # Only require fftw3 if found (non-2^ size image processing fails without it) + # We do not toggle cimg_use_fftw3, it is buggy + if pkgconfig.exists("fftw3"): + define_macros += [("cimg_use_fftw3", None)] + pkgconfig_list += ["fftw3"] + if sys.platform not in ("cygwin", "win32", "msys"): + extra_link_args += ["-lfftw3_threads"] -library_dirs = packages["library_dirs"] + [here, gmic_src_path] -if sys.platform == "darwin": - library_dirs += ["/usr/local/opt/llvm@6/lib"] -include_dirs = packages["include_dirs"] + [here, gmic_src_path] -if sys.platform == "darwin": - include_dirs += ["/usr/local/opt/llvm@6/include"] -# Debugging is now set through --global-option --debug and more. -# debugging_args = [ -# "-O0", -# "-g", -# ] # Uncomment this for faster compilation with debug symbols and no optimization + # Only compile with OpenCV if exists (nice for the 'camera' G'MIC command :-D ) + if pkgconfig.exists("opencv"): + define_macros += [("cimg_use_opencv", None)] + pkgconfig_list += ["opencv"] -debug_enabled = "--debug" in sys.argv + if pkgconfig.exists("libcurl"): + define_macros += [("cimg_use_curl", None)] + pkgconfig_list += ["libcurl"] + + packages = pkgconfig.parse(" ".join(pkgconfig_list)) + libraries = packages["libraries"] + [ + "pthread" + ] # removed core-dumping 'gomp' temporarily (for manylinux builds) -extra_compile_args = ["-std=c++11"] -if debug_enabled: - extra_compile_args += ["-O0"] - extra_compile_args += ["-g"] + library_dirs = packages["library_dirs"] + [here, gmic_src_path] + if sys.platform == "darwin": + library_dirs += ["/usr/local/opt/llvm@6/lib"] + include_dirs = packages["include_dirs"] + [here, gmic_src_path] + if sys.platform == "darwin": + include_dirs += ["/usr/local/opt/llvm@6/include"] + # Debugging is now set through --global-option --debug and more. + # debugging_args = [ + # "-O0", + # "-g", + # ] # Uncomment this for faster compilation with debug symbols and no optimization else: - extra_compile_args += ["-Ofast", "-flto"] - extra_link_args += ["-flto"] + # WINDOWS build flags generation + x = "x64" if sys.maxsize > 2 ** 32 else "x86" + vcpkg_lib_dir = os.path.join( + os.environ["VCPKG_INSTALLATION_ROOT"], "installed", f"{x}-windows", "lib" + ) + libraries = ["fftw3", "libpng16", "jpeg", "curl", "zlib", "tiff"] + library_dirs = [vcpkg_lib_dir, gmic_src_path] + include_dirs = [here, gmic_src_path] + define_macros = [] + define_macros += [("cimg_use_curl", None)] + define_macros += [("cimg_use_fftw3", None)] + define_macros += [("cimg_display", None)] + define_macros += [("cimg_use_jpeg", None)] + define_macros += [("cimg_use_tiff", None)] + define_macros += [("cimg_use_png", None)] + define_macros += [("cimg_display", None)] + extra_compile_args.extend(["/std:c11", "/OpenMP"]) if sys.platform == "darwin": extra_compile_args += ["-fopenmp", "-stdlib=libc++"] @@ -104,26 +136,49 @@ "-nodefaultlibs", "-lc++", ] # options inspired by https://github.com/explosion/spaCy/blob/master/setup.py -elif sys.platform == "linux": # Enable openmp for 32bit & 64bit linuxes +elif not IS_WINDOWS: + # Enable openmp for 32bit & 64bit linuxes and posix'ed windows extra_compile_args += ["-fopenmp"] extra_link_args += ["-lgomp"] +package_data = {} +# Force Windows and Windows posix'ed platforms as Windows-like for CImg/G'MIC +# Also fix libcurl include and lib paths, which are not correctly mapped by pkg-config +if sys.platform in ("msys", "cygwin", "win32"): + define_macros.append(("cimg_OS", "2")) + package_data = {"gmic": ["win_dll/*.dll"]} + +# # MSYS2 / Github Action hack to fix libcurl +# extra_compile_args += ["-ID:/a/_temp/msys/msys64/usr/include"] +# extra_link_args += ["-LD:/a/_temp/msys/msys64/usr/lib"] + + print("Define macros:") print(define_macros) -# Static CPython gmic.so embedding libgmic.so.2 -gmic_module = Extension( - "gmic", +extension_kwargs = dict( + name="gmic", include_dirs=include_dirs, libraries=libraries, library_dirs=library_dirs, - sources=["gmicpy.cpp", path.join(gmic_src_path, "gmic.cpp")], + sources=[ + "gmicpy.cpp", + path.join(gmic_src_path, "gmic.cpp"), + ], define_macros=define_macros, extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, language="c++", ) + +print("Extension options:") +print(extension_kwargs) + + +# Static CPython gmic.so embedding libgmic.so / .dll +gmic_module = Extension(**extension_kwargs) + # Get the long description from the README file with open(path.join(here, "README.md"), encoding="utf-8") as f: long_description = f.read() @@ -136,64 +191,31 @@ # Arguments marked as "Required" below must be included for upload to PyPI. # Fields marked as "Optional" may be commented out. + +def get_package_data(): + """Return a list of DLL dependencies paths if on Windows platforms and DLL dir was prepared + Needs build_tools.bash 4b_build_windows_portable_wheel to be run first on Windows OS + Returns nothing quietly on other platforms + """ + package_data = {} + if sys.platform in ("msys", "cygwin", "win32") and path.exists(WIN_DLL_DIR): + libfiles = listdir(WIN_DLL_DIR) + package_data["win_dll"] = libfiles + + # TODO remove me + print("just built package_data:", package_data) + return package_data + + setup( - # This is the name of your project. The first time you publish this - # package, this name will be registered for you. It will determine how - # users can install this project, e.g.: - # - # $ pip install sampleproject - # - # And where it will live on PyPI: https://pypi.org/project/sampleproject/ - # - # There are some restrictions on what makes a valid project name - # specification here: - # https://packaging.python.org/specifications/core-metadata/#name - name="gmic", # Required - # Versions should comply with PEP 440: - # https://www.python.org/dev/peps/pep-0440/ - # - # For a discussion on single-sourcing the version across setup.py and the - # project code, see - # https://packaging.python.org/en/latest/single_source_version.html + name="gmic", version=version, - # This is a one-line description or tagline of what your project does. This - # corresponds to the "Summary" metadata field: - # https://packaging.python.org/specifications/core-metadata/#summary description="Binary Python3 bindings for the G'MIC C++ image processing library", # Optional - # This is an optional longer description of your project that represents - # the body of text which users will see when they visit PyPI. - # - # Often, this is the same as your README, so you can just read it in from - # that file directly (as we have already done above) - # - # This field corresponds to the "Description" metadata field: - # https://packaging.python.org/specifications/core-metadata/#description-optional - long_description=long_description, # Optional - # Denotes that our long_description is in Markdown; valid values are - # text/plain, text/x-rst, and text/markdown - # - # Optional if long_description is written in reStructuredText (rst) but - # required for plain-text or Markdown; if unspecified, "applications should - # attempt to render [the long_description] as text/x-rst; charset=UTF-8 and - # fall back to text/plain if it is not valid rst" (see link below) - # - # This field corresponds to the "Description-Content-Type" metadata field: - # https://packaging.python.org/specifications/core-metadata/#description-content-type-optional + long_description=long_description, long_description_content_type="text/markdown", # Optional (see note above) - # This should be a valid link to your project's main homepage. - # - # This field corresponds to the "Home-Page" metadata field: - # https://packaging.python.org/specifications/core-metadata/#home-page-optional url="https://github.com/dtschump/gmic-py", # Optional - # This should be your name or the name of the organization which owns the - # project. author="David Tschumperlé, Jonathan-David Schröder G'MIC GREYC IMAGE Team of CNRS, France", # Optional - # This should be a valid email address corresponding to the author listed - # above. author_email="David.Tschumperle@ensicaen.fr, jonathan.schroder@gmail.com", # Optional - # Classifiers help users find your project by categorizing it. - # - # For a list of valid classifiers, see https://pypi.org/classifiers/ classifiers=[ # Optional # How mature is this project? Common values are # 3 - Alpha @@ -215,6 +237,7 @@ "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ], # This field adds keywords for your project which will appear on the # project page. What does your project relate to? @@ -266,9 +289,9 @@ # # If using Python 2.6 or earlier, then these have to be included in # MANIFEST.in as well. - # package_data={ # Optional - # 'sample': ['package_data.dat'], - # }, + package_data=package_data, + # include_package_data=True, + # data_files=[("", ["COPYING"])], # Although 'package_data' is the preferred approach, in some case you may # need to place data files outside of your packages. See: # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files diff --git a/test-requirements-win.txt b/test-requirements-win.txt new file mode 100644 index 00000000..779be7cc --- /dev/null +++ b/test-requirements-win.txt @@ -0,0 +1,10 @@ +# For testing, pip install -r dev-requirements.txt first, then this + +#scikit-image +numpy #==1.16.1 # fixes https://github.com/scikit-image/scikit-image/issues/3655 +Pillow +pytest +setuptools +pytest-xdist #allow parallelization +# psutil # won't compile easily enough on MSYS2 WINDOWS +# Do not put the 'black' Python code formatter as a dependency here it will not install for Python <= 3.6 and crash the Mac OS CI, black must be installed at formatting check time, outside any docker wheel-building environment. See the related formatting functions in build_tools.bash script. diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 00000000..b47fd3d8 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,10 @@ +# For testing, pip install -r dev-requirements.txt first, then this + +#scikit-image +numpy #==1.16.1 # fixes https://github.com/scikit-image/scikit-image/issues/3655 +Pillow +pytest +setuptools +pytest-xdist #allow parallelization +psutil +# Do not put the 'black' Python code formatter as a dependency here it will not install for Python <= 3.6 and crash the Mac OS CI, black must be installed at formatting check time, outside any docker wheel-building environment. See the related formatting functions in build_tools.bash script. diff --git a/tests/.gitignore b/tests/.gitignore index 8e0855fa..297a2419 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,2 +1,2 @@ __pycache__ -*.png +./*.png diff --git a/tests/samples/apples.png b/tests/samples/apples.png new file mode 100644 index 00000000..e678d19f Binary files /dev/null and b/tests/samples/apples.png differ diff --git a/tests/samples/duck.png b/tests/samples/duck.png new file mode 100644 index 00000000..395d07e6 Binary files /dev/null and b/tests/samples/duck.png differ diff --git a/tests/samples/lena.png b/tests/samples/lena.png new file mode 100644 index 00000000..93e982bc Binary files /dev/null and b/tests/samples/lena.png differ diff --git a/tests/samples/leno.png b/tests/samples/leno.png new file mode 100644 index 00000000..e0a47b67 Binary files /dev/null and b/tests/samples/leno.png differ diff --git a/tests/samples/parrots.png b/tests/samples/parrots.png new file mode 100644 index 00000000..f07c5dbf Binary files /dev/null and b/tests/samples/parrots.png differ diff --git a/tests/test_gmic_numpy.py b/tests/test_gmic_numpy.py index 4b20c7c4..6847fa5c 100644 --- a/tests/test_gmic_numpy.py +++ b/tests/test_gmic_numpy.py @@ -74,7 +74,7 @@ def test_gmic_image_to_numpy_ndarray_exception_on_unimportable_numpy_module( import gmic images = [] - gmic_instance_run(images=images, command="sp duck") + gmic_instance_run(images=images, command="tests/samples/duck.png") with pytest.raises( gmic.GmicException, match=r".*'numpy' module cannot be imported.*" ): @@ -91,7 +91,10 @@ def test_gmic_image_to_numpy_ndarray_exception_on_unimportable_numpy_module( @pytest.mark.parametrize(**squeeze_toggles) @pytest.mark.parametrize( "gmic_command", - ["sp apples", """3,5,7,2,'x*cos(0.5236)+y*sin(0.8)' -normalize 0,255"""], + [ + "tests/samples/apples.png", + """3,5,7,2,'x*cos(0.5236)+y*sin(0.8)' -normalize 0,255""", + ], ids=["2dsample", "3dsample"], ) def test_gmic_image_to_numpy_helper_fuzzying( @@ -162,7 +165,7 @@ def test_gmic_image_to_numpy_ndarray_basic_attributes(gmic_instance_run): import numpy single_image_list = [] - gmic_instance_run(images=single_image_list, command="sp apples") + gmic_instance_run(images=single_image_list, command="tests/samples/apples.png") gmic_image = single_image_list[0] # we do not interleave to keep the same data structure for later comparison numpy_image = gmic_image.to_numpy_helper(interleave=False) @@ -181,8 +184,8 @@ def test_gmic_image_to_numpy_ndarray_basic_attributes(gmic_instance_run): @pytest.mark.parametrize(**gmic_instance_types) def test_in_memory_gmic_image_to_numpy_nd_array_to_gmic_image(gmic_instance_run): single_image_list = [] - gmic_instance_run(images=single_image_list, command="sp duck") - # TODO convert back and compare with original sp duck GmicImage + gmic_instance_run(images=single_image_list, command="tests/samples/duck.png") + # TODO convert back and compare with original tests/samples/duck.png GmicImage @pytest.mark.parametrize(**gmic_instance_types) @@ -195,7 +198,7 @@ def test_numpy_ndarray_RGB_2D_image_gmic_run_without_gmicimage_wrapping( im1_name = "image.png" im2_name = "image.png" - gmic_instance_run("sp duck output " + im1_name) + gmic_instance_run("tests/samples/duck.png output " + im1_name) np_PIL_image = numpy.array(PIL.Image.open(im1_name)) # TODO line below must fail because single numpy arrays rewrite is impossible for us with pytest.raises( @@ -218,7 +221,7 @@ def test_numpy_ndarray_RGB_2D_image_integrity_through_numpyPIL_or_gmicimage_from im2_name = "image2.bmp" # 1. Generate duck bitmap, save it to disk - gmic_instance_run("sp duck -output " + im1_name) + gmic_instance_run("tests/samples/duck.png -output " + im1_name) # 2. Load disk duck through PIL/numpy, make it a GmicImage image_from_numpy = numpy.array(PIL.Image.open(im1_name)) @@ -232,7 +235,7 @@ def test_numpy_ndarray_RGB_2D_image_integrity_through_numpyPIL_or_gmicimage_from # 3. Load duck into a regular GmicImage through G'MIC without PIL/numpy imgs = [] - gmic_instance_run(images=imgs, command="sp duck") + gmic_instance_run(images=imgs, command="tests/samples/duck.png") gmicimage_from_gmic = imgs[0] # 4. Use G'MIC to compare both duck GmicImages from numpy and gmic sources @@ -248,7 +251,7 @@ def test_numpy_PIL_modes_to_gmic(gmic_instance_run): origin_image_name = "a.bmp" gmicimages = [] - gmic_instance_run("sp duck output " + origin_image_name) + gmic_instance_run("tests/samples/duck.png output " + origin_image_name) PILimage = PIL.Image.open("a.bmp") modes = [ @@ -292,7 +295,7 @@ def test_numpy_PIL_modes_to_gmic(gmic_instance_run): def test_basic_from_numpy_helper_to_numpy_helper(): duck = [] - gmic.run("sp duck", duck) + gmic.run("tests/samples/duck.png", duck) original_duck_gmic_image = duck[0] duck_numpy_image = original_duck_gmic_image.to_numpy_helper(squeeze_shape=True) duck_io_gmic_image = gmic.GmicImage.from_numpy_helper(duck_numpy_image) @@ -316,7 +319,7 @@ def test_from_numpy_helper_proper_dimensions_number(): def test_basic_to_numpy_helper_from_numpy_helper(): - gmic.run("sp duck output duck.png") + gmic.run("tests/samples/duck.png output duck.png") import PIL.Image pil_image = numpy.array(PIL.Image.open("duck.png")) diff --git a/tests/test_gmic_numpy_toolkits.py b/tests/test_gmic_numpy_toolkits.py index 27b235e8..6b674482 100644 --- a/tests/test_gmic_numpy_toolkits.py +++ b/tests/test_gmic_numpy_toolkits.py @@ -74,7 +74,7 @@ def numpy_PIL_duck(): im1_name = "image.bmp" # 1. Generate duck bitmap, save it to disk - gmic.run("sp duck -output " + im1_name) + gmic.run("tests/samples/duck.png -output " + im1_name) # 2. Load disk duck through PIL/numpy, make it a GmicImage yield numpy.array(PIL.Image.open(im1_name)) @@ -310,7 +310,7 @@ def test_toolkit_to_PIL(): l = [] PIL_leno_filename = "PIL_leno.png" PIL_apples_filename = "PIL_apples.png" - gmic.run("sp leno sp apples", l) + gmic.run("tests/samples/leno.png tests/samples/apples.png", l) PIL_leno, PIL_apples = l[0].to_PIL(), l[1].to_PIL() PIL_leno.save(PIL_leno_filename, compress_level=0) PIL_apples.save(PIL_apples_filename, compress_level=0) @@ -325,7 +325,7 @@ def test_toolkit_to_PIL(): def test_toolkit_to_PIL_advanced(): # to_PIL fine-graining parameters testing l = [] - gmic.run("sp leno sp apples", l) + gmic.run("tests/samples/leno.png tests/samples/apples.png", l) PIL_leno = l[0].to_PIL(mode="HSV") assert PIL_leno.mode == "HSV" assert PIL_leno.width == l[0]._width @@ -342,18 +342,18 @@ def test_toolkit_from_PIL(): import PIL.Image l = [] - gmic.run("sp leno sp apples", l) + gmic.run("tests/samples/leno.png tests/samples/apples.png", l) PIL_apples_filename = "PIL_apples.png" PIL_leno_filename = "PIL_leno.png" - gmic.run("sp leno output " + PIL_leno_filename) + gmic.run("tests/samples/leno.png output " + PIL_leno_filename) PIL_leno = PIL.Image.open(PIL_leno_filename) leno = gmic.GmicImage.from_PIL(PIL_leno) assert_gmic_images_are_identical(l[0], leno) assert_non_empty_file_exists(PIL_leno_filename).unlink() - gmic.run("sp apples output " + PIL_apples_filename) + gmic.run("tests/samples/apples.png output " + PIL_apples_filename) PIL_apples = PIL.Image.open(PIL_apples_filename) apples = gmic.GmicImage.from_PIL(PIL_apples) assert_gmic_images_are_identical(l[1], apples) diff --git a/tests/test_gmic_py.py b/tests/test_gmic_py.py index f5d85aba..be1f659d 100644 --- a/tests/test_gmic_py.py +++ b/tests/test_gmic_py.py @@ -73,7 +73,9 @@ def test_run_gmic_ensure_openmp_linked_and_working(capfd, gmic_instance_run): import traceback import sys - gmic_instance_run("v - sp lena eval. \"end(run('echo_stdout[] ',merge(t,max)))\"") + gmic_instance_run( + "v - tests/samples/lena.png eval. \"end(run('echo_stdout[] ',merge(t,max)))\"" + ) outerr = capfd.readouterr() try: assert ( @@ -104,7 +106,7 @@ def test_run_gmic_instance_run_helloworld(capfd, gmic_instance_run): def make_sample_format_file(extension): filename = "{}_format_io_test.{}".format(extension, extension) - gmic.run("sp apples output {}".format(filename)) + gmic.run("tests/samples/apples.png output {}".format(filename)) yield filename @@ -250,7 +252,7 @@ def test_gmic_user_file_autoload_and_use(gmic_instance_run, request, capfd): gmic_instance_run = g.run # 2. Check that our custom command was auto-loaded as expected - gmic_instance_run("sp leno " + gmicpy_testing_command) + gmic_instance_run("tests/samples/leno.png " + gmicpy_testing_command) @pytest.mark.parametrize(**gmic_instance_types) @@ -269,9 +271,9 @@ def test_gmic_user_file_explicit_load_and_use(gmic_instance_run, capfd): gmicpy_testing_command ), ): - gmic_instance_run("sp leno " + gmicpy_testing_command) + gmic_instance_run("tests/samples/leno.png " + gmicpy_testing_command) else: - gmic_instance_run("sp leno " + gmicpy_testing_command) + gmic_instance_run("tests/samples/leno.png " + gmicpy_testing_command) # Skipping test per https://github.com/dtschump/gmic/issues/256#issuecomment-694121423 @@ -513,7 +515,7 @@ def assert_gmic_image_is_filled_with(gmic_image, w, h, d, s, pixel_value): def test_gmic_image_pixel_access(): images = [] - gmic.run("sp apples", images) + gmic.run("tests/samples/apples.png", images) image = images[0] assert image(0, 0, 0, 0) == image() assert image(s=3) == image(0, 0, 0, 3) @@ -1070,7 +1072,7 @@ def test_gmic_class_direct_run_remains_usable_instance(): def test_filling_empty_gmicimage_list_with_input_image_nonregtest_issue_30(): images = [] - gmic.run(images=images, command="sp lena") + gmic.run(images=images, command="tests/samples/lena.png") assert len(images) == 1 assert type(images[0]) == gmic.GmicImage @@ -1078,7 +1080,7 @@ def test_filling_empty_gmicimage_list_with_input_image_nonregtest_issue_30(): def test_gmic_module_run_vs_single_instance_run_benchmark(): from time import time - testing_command = "sp lena blur 10 blur 30 blur 4" + testing_command = "tests/samples/lena.png blur 10 blur 30 blur 4" testing_iterations_max = 100 expected_speed_improvement_by_single_instance_runs = 1.1 # at least 10% diff --git a/tests/test_gmic_py_filters_io.py b/tests/test_gmic_py_filters_io.py index 0568502d..3a04c6af 100644 --- a/tests/test_gmic_py_filters_io.py +++ b/tests/test_gmic_py_filters_io.py @@ -305,7 +305,7 @@ def test_gmic_filter_io(filter_inputs, filter_id, filter_params): ): filter_params[pos] = '"{}"'.format(param.replace('"', '\\"')) # One of the worst CLI expression that works thanks to escaping similar to the one below: - # gmic sp parrots -srand 781123 fx_watermark_fourier \"\(a\) b\'o\\\"nsoir\",53 output[0] "/home/jd/Productions/GMIC/gmic-py/build/lib.linux-x86_64-3.7/test-images/fx_watermark_fourier_gmiccli.png" + # gmic tests/samples/parrots.png -srand 781123 fx_watermark_fourier \"\(a\) b\'o\\\"nsoir\",53 output[0] "/home/jd/Productions/GMIC/gmic-py/build/lib.linux-x86_64-3.7/test-images/fx_watermark_fourier_gmiccli.png" filter_params_cli[pos] = '\\"{}\\"'.format( param.replace("(", "\\(") .replace(")", "\\)") diff --git a/tests/test_gmic_py_memfreeing.py b/tests/test_gmic_py_memfreeing.py index bf6e043c..50b35a1f 100644 --- a/tests/test_gmic_py_memfreeing.py +++ b/tests/test_gmic_py_memfreeing.py @@ -1,6 +1,13 @@ import pytest import numpy -import psutil + +try: + import psutil +except: + pytest.skip( + "psutil not found - this is OK for a mingw environment - see https://github.com/giampaolo/psutil/blob/master/INSTALL.rst#windows", + allow_module_level=True, + ) import os import struct diff --git a/wheel_repair.py b/wheel_repair.py new file mode 100644 index 00000000..6f7c6ce8 --- /dev/null +++ b/wheel_repair.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python + +import os +import sys +import shutil +import hashlib +import zipfile +import argparse +import tempfile +from collections import defaultdict + +import pefile +from machomachomangler.pe import redll + + +def hash_filename(filepath, blocksize=65536): + hasher = hashlib.sha256() + + with open(filepath, "rb") as afile: + buf = afile.read(blocksize) + while len(buf) > 0: + hasher.update(buf) + buf = afile.read(blocksize) + + root, ext = os.path.splitext(filepath) + return f"{os.path.basename(root)}-{hasher.hexdigest()[:8]}{ext}" + + +def find_dll_dependencies(dll_filepath, vcpkg_bin_dir): + pe = pefile.PE(dll_filepath) + + for entry in pe.DIRECTORY_ENTRY_IMPORT: + entry_name = entry.dll.decode("utf-8") + print("debug: attempting to find", entry, entry.dll, entry.imports, entry.struct, "in", vcpkg_bin_dir) + if entry_name in os.listdir(vcpkg_bin_dir): + dll_dependencies[os.path.basename(dll_filepath)].add(entry_name) + _dll_filepath = os.path.join(vcpkg_bin_dir, entry_name) + find_dll_dependencies(_dll_filepath, vcpkg_bin_dir) + + +def mangle_filename(old_filename, new_filename, mapping): + with open(old_filename, "rb") as f: + buf = f.read() + + try: + new_buf = redll(buf, mapping) + print("Building mangle filename mapping and redll:", old_filename, new_filename, mapping) + + with open(new_filename, "wb") as f: + f.write(new_buf) + except ValueError: + print("Unsolvable machomangler buffer length error. Skipping mangling of:", old_filename) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Vendor in external shared library dependencies of a wheel." + ) + + parser.add_argument("WHEEL_FILE", type=str, help="Path to wheel file") + parser.add_argument( + "-d", "--dll-dir", dest="DLL_DIR", type=str, help="Directory to find the DLLs" + ) + parser.add_argument( + "-w", + "--wheel-dir", + dest="WHEEL_DIR", + type=str, + help=('Directory to store delocated wheels (default: "wheelhouse/")'), + default="wheelhouse/", + ) + + args = parser.parse_args() + + wheel_name = os.path.basename(args.WHEEL_FILE) + package_name = wheel_name.split("-")[0] + repaired_wheel = os.path.join(args.WHEEL_DIR, wheel_name) + + old_wheel_dir = tempfile.mkdtemp() + new_wheel_dir = tempfile.mkdtemp() + + with zipfile.ZipFile(args.WHEEL_FILE, "r") as wheel: + wheel.extractall(old_wheel_dir) + print("showing old_wheel_dir contents") + print(os.listdir(old_wheel_dir)) + print("attempting stat of old_wheel_dir:", old_wheel_dir) + os.stat(old_wheel_dir) + wheel.extractall(new_wheel_dir) + pe_path = list(filter(lambda x: x.endswith((".pyd", ".dll")), wheel.namelist()))[0] + print("debug: pe_path is:", pe_path) + #tmp_pe_path = os.path.join(old_wheel_dir, package_name, os.path.basename(pe_path)) + # add dll recursive autodetection + tmp_pe_path = os.path.join(old_wheel_dir, os.path.basename(pe_path)) + print("debug: tmp_pe_path is:", tmp_pe_path) + + # https://docs.python.org/3/library/platform.html#platform.architecture + x = "x64" if sys.maxsize > 2**32 else "x86" + # set VCPKG_INSTALLATION_ROOT=C:\dev\vcpkg + #dll_dir = os.path.join(os.environ["VCPKG_INSTALLATION_ROOT"], "installed", f"{x}-windows", "bin") + + dll_dir = args.DLL_DIR + if not dll_dir and os.environ["VCPKG_INSTALLATION_ROOT"]: + # https://docs.python.org/3/library/platform.html#platform.architecture + #dll_dir = os.path.join(os.environ["VCPKG_INSTALLATION_ROOT"], "installed", f"{x}-windows", "bin") + x = "x64" if sys.maxsize > 2**32 else "x86" + # set VCPKG_INSTALLATION_ROOT=C:\dev\vcpkg + + print("debug: dll_dir (lookup directory) is:", dll_dir) + + dll_dependencies = defaultdict(set) + find_dll_dependencies(tmp_pe_path, dll_dir) + print("dll_dependencies found:", dll_dependencies) + + + new_wheel_libs_dir = os.path.join(new_wheel_dir, package_name) + os.makedirs(new_wheel_dir, exist_ok=True) + + for dll, dependencies in dll_dependencies.items(): + print("about to mangle:", dll) + mapping = {} + + for dep in dependencies: + print("about to hash:",os.path.join(dll_dir, dep)) + hashed_name = hash_filename(os.path.join(dll_dir, dep)) # already basename + mapping[dep.encode("ascii")] = hashed_name.encode("ascii") + shutil.copy( + os.path.join(dll_dir, dep), + os.path.join(new_wheel_libs_dir, hashed_name), + ) + + if dll == pe_path: + # skip mangling module's portable executable itself + continue + elif dll.endswith(".pyd"): + old_name = os.path.join( + new_wheel_libs_dir, os.path.basename(tmp_pe_path) + ) + new_name = os.path.join( + new_wheel_libs_dir, os.path.basename(tmp_pe_path) + ) + elif dll.endswith(".dll"): + old_name = os.path.join(dll_dir, dll) + print("about to hash:",os.path.join(dll_dir, dll)) + hashed_name = hash_filename(os.path.join(dll_dir, dll)) # already basename + new_name = os.path.join(new_wheel_libs_dir, hashed_name) + + mangle_filename(old_name, new_name, mapping) + + with zipfile.ZipFile(repaired_wheel, "w", zipfile.ZIP_DEFLATED) as new_wheel: + for root, dirs, files in os.walk(new_wheel_dir): + for file in files: + print("new wheel copying:", os.path.join(root, file), os.path.join(os.path.basename(root), file)) + new_wheel.write( + os.path.join(root, file), os.path.join(os.path.basename(root), file) + ) diff --git a/win_dll/.gitignore b/win_dll/.gitignore new file mode 100644 index 00000000..e69de29b