From 04224f1b655dad750b81f9476f25abff3c008b29 Mon Sep 17 00:00:00 2001 From: Robert Oostenveld Date: Sun, 23 Apr 2023 13:17:46 +0200 Subject: [PATCH 01/13] Add hook for pylsl --- .../hooks/stdhooks/hook-pylsl.py | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py diff --git a/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py b/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py new file mode 100644 index 00000000..4a13f777 --- /dev/null +++ b/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py @@ -0,0 +1,81 @@ +import os +import platform +import struct +import pathlib +from ctypes import util +from PyInstaller.utils.hooks import get_module_file_attribute, logger + +""" +Hook for pylsl, i.e., the Python wrapper around labstreaminglayer (aka LSL). +The pylsl package loads the C++ liblsl library using CDLL. The logic for +locating liblsl is replicated here; subsequently it is added to the binaries +and packaged in the compiled version such that pylsl will be able to find it +there as well. + +See also +- https://labstreaminglayer.readthedocs.io +- https://github.com/labstreaminglayer/pylsl +- https://github.com/sccn/liblsl +""" + + +def find_liblsl_library(): + """finds the binary lsl library. + + Search order is to first try to use the path stored in the environment + variable PYLSL_LIB (if available), then search through the package + directory, and finally search the whole system. + """ + + # first try the environment variable PYLSL_LIB + if "PYLSL_LIB" in os.environ: + libfile = os.environ["PYLSL_LIB"] + if libfile and os.path.isfile(libfile): + return libfile + + # this will only run if PYLSL_LIB did not resolve the library + os_name = platform.system() + if os_name in ["Windows", "Microsoft"]: + libsuffix = ".dll" + elif os_name == "Darwin": + libsuffix = ".dylib" + elif os_name == "Linux": + libsuffix = ".so" + else: + raise RuntimeError("unrecognized operating system:", os_name) + + for scope in ["package", "system"]: + for libprefix in ["", "lib"]: + for debugsuffix in ["", "-debug"]: + for bitness in ["", str(8 * struct.calcsize("P"))]: + if scope == "package": + # try in the package directory at pylsl/lib + module_dir = pathlib.Path(get_module_file_attribute('pylsl')).parent + libfile = os.path.join(module_dir, libprefix + "lsl" + bitness + debugsuffix + libsuffix) + if libfile and os.path.isfile(libfile): + return libfile + elif (scope == "system") and os_name not in ["Windows", "Microsoft"]: + # try on the system library path, this includes the conda library path + libfile = util.find_library(libprefix + "lsl" + bitness + debugsuffix) + if libfile and os.path.isfile(libfile): + return libfile + elif (scope == "system") and os_name == "Darwin": + # try on the homebrew library path + libfile = util.find_library('/opt/homebrew/lib/' + libprefix + "lsl" + bitness + debugsuffix) + if libfile and os.path.isfile(libfile): + return libfile + + +libfile = find_liblsl_library() + +if libfile: + # add the liblsl library to the binaries + # it gets packaged in pylsl/lib, which is where pylsl will look first + binaries = [(libfile, os.path.join('pylsl', 'lib'))] +else: + logger.warning("liblsl shared library not found - pylsl will likely fail to work!") + binaries = [] + + +if __name__ == '__main__': + print(binaries) From db9553ac6d0a38445216a9ca47d45997c34c3199 Mon Sep 17 00:00:00 2001 From: Robert Oostenveld Date: Sun, 23 Apr 2023 13:24:24 +0200 Subject: [PATCH 02/13] Add a news entry --- news/573.new.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/573.new.rst diff --git a/news/573.new.rst b/news/573.new.rst new file mode 100644 index 00000000..00318afe --- /dev/null +++ b/news/573.new.rst @@ -0,0 +1 @@ +Add hook for ``pylsl`` From 870e8103231d5c06df8e2ff29a00a3dd258440b5 Mon Sep 17 00:00:00 2001 From: Robert Oostenveld Date: Sun, 23 Apr 2023 13:31:30 +0200 Subject: [PATCH 03/13] remove the __main__, which was only there for testing --- src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py b/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py index 4a13f777..8e1cc731 100644 --- a/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py +++ b/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py @@ -75,7 +75,3 @@ def find_liblsl_library(): else: logger.warning("liblsl shared library not found - pylsl will likely fail to work!") binaries = [] - - -if __name__ == '__main__': - print(binaries) From d2bd63ffc5df9b4e0204495346a55ec96fc6e366 Mon Sep 17 00:00:00 2001 From: Robert Oostenveld Date: Sun, 23 Apr 2023 18:12:14 +0200 Subject: [PATCH 04/13] Use the find_liblsl_libraries() generatorm from pylsl itself --- .../hooks/stdhooks/hook-pylsl.py | 87 +++++-------------- 1 file changed, 22 insertions(+), 65 deletions(-) diff --git a/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py b/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py index 8e1cc731..ff8aee75 100644 --- a/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py +++ b/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py @@ -1,72 +1,29 @@ -import os -import platform -import struct -import pathlib -from ctypes import util -from PyInstaller.utils.hooks import get_module_file_attribute, logger - -""" -Hook for pylsl, i.e., the Python wrapper around labstreaminglayer (aka LSL). -The pylsl package loads the C++ liblsl library using CDLL. The logic for -locating liblsl is replicated here; subsequently it is added to the binaries -and packaged in the compiled version such that pylsl will be able to find it -there as well. - -See also -- https://labstreaminglayer.readthedocs.io -- https://github.com/labstreaminglayer/pylsl -- https://github.com/sccn/liblsl -""" - +# ------------------------------------------------------------------ +# Copyright (c) 2023 PyInstaller Development Team. +# +# This file is distributed under the terms of the GNU General Public +# License (version 2.0 or later). +# +# The full license is available in LICENSE.GPL.txt, distributed with +# this software. +# +# SPDX-License-Identifier: GPL-2.0-or-later +# ------------------------------------------------------------------ -def find_liblsl_library(): - """finds the binary lsl library. - - Search order is to first try to use the path stored in the environment - variable PYLSL_LIB (if available), then search through the package - directory, and finally search the whole system. - """ - - # first try the environment variable PYLSL_LIB - if "PYLSL_LIB" in os.environ: - libfile = os.environ["PYLSL_LIB"] - if libfile and os.path.isfile(libfile): - return libfile - - # this will only run if PYLSL_LIB did not resolve the library - os_name = platform.system() - if os_name in ["Windows", "Microsoft"]: - libsuffix = ".dll" - elif os_name == "Darwin": - libsuffix = ".dylib" - elif os_name == "Linux": - libsuffix = ".so" - else: - raise RuntimeError("unrecognized operating system:", os_name) +import os +from PyInstaller.utils.hooks import logger - for scope in ["package", "system"]: - for libprefix in ["", "lib"]: - for debugsuffix in ["", "-debug"]: - for bitness in ["", str(8 * struct.calcsize("P"))]: - if scope == "package": - # try in the package directory at pylsl/lib - module_dir = pathlib.Path(get_module_file_attribute('pylsl')).parent - libfile = os.path.join(module_dir, libprefix + "lsl" + bitness + debugsuffix + libsuffix) - if libfile and os.path.isfile(libfile): - return libfile - elif (scope == "system") and os_name not in ["Windows", "Microsoft"]: - # try on the system library path, this includes the conda library path - libfile = util.find_library(libprefix + "lsl" + bitness + debugsuffix) - if libfile and os.path.isfile(libfile): - return libfile - elif (scope == "system") and os_name == "Darwin": - # try on the homebrew library path - libfile = util.find_library('/opt/homebrew/lib/' + libprefix + "lsl" + bitness + debugsuffix) - if libfile and os.path.isfile(libfile): - return libfile +try: + # the import will fail it the library cannot be found + from pylsl import pylsl + # the find_liblsl_libraries() is a generator function that yields multiple possibilities + for libfile in pylsl.find_liblsl_libraries(): + if libfile: + break +except: + libfile = None -libfile = find_liblsl_library() if libfile: # add the liblsl library to the binaries From 20fdd6e75f57c7c6bba578224838921a406a9878 Mon Sep 17 00:00:00 2001 From: Robert Oostenveld Date: Sun, 23 Apr 2023 18:12:49 +0200 Subject: [PATCH 05/13] added test, specific for macOS using brew to install the library --- .github/workflows/pr-test.yml | 2 ++ requirements-test-libraries.txt | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 8179d2f6..a371bd17 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -98,6 +98,8 @@ jobs: brew install pango # Install libdiscid (dependency of discid python package). brew install libdiscid + # Install lsl library for pylsl + brew install lsl - name: Install dependencies shell: bash diff --git a/requirements-test-libraries.txt b/requirements-test-libraries.txt index e0122ae2..9a17d3a8 100644 --- a/requirements-test-libraries.txt +++ b/requirements-test-libraries.txt @@ -122,7 +122,6 @@ minecraft-launcher-lib==5.3; python_version >= '3.8' scikit-learn==1.2.2; python_version >= '3.8' scikit-image==0.20.0; python_version >= '3.8' - # ------------------- Platform (OS) specifics # PyEnchant only pre-builds macOS and Windows @@ -140,5 +139,8 @@ pywin32-ctypes==0.2.0; sys_platform == 'win32' # pymediainfo on linux does not bundle mediainfo shared library, and requires system one. pymediainfo==6.0.1; sys_platform == 'darwin' or sys_platform == 'win32' +# the required library can be installed with "brew install lsl" on macOS, or with "conda install liblsl" on any platform +pylsl==1.16.1; sys_platform == "darwin" + # Include the requirements for testing -r requirements-test.txt From ae02cee4800004cbc189f178f9aaec5a3318a692 Mon Sep 17 00:00:00 2001 From: Robert Oostenveld Date: Thu, 27 Apr 2023 13:32:16 +0200 Subject: [PATCH 06/13] ENH - import pylsl in a function and call it from an isolated subprocess --- .../hooks/stdhooks/hook-pylsl.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py b/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py index ff8aee75..6db30453 100644 --- a/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py +++ b/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py @@ -11,19 +11,23 @@ # ------------------------------------------------------------------ import os -from PyInstaller.utils.hooks import logger +from PyInstaller.utils.hooks import logger, isolated -try: - # the import will fail it the library cannot be found - from pylsl import pylsl - - # the find_liblsl_libraries() is a generator function that yields multiple possibilities - for libfile in pylsl.find_liblsl_libraries(): - if libfile: - break -except: - libfile = None +def find_library(): + try: + # the import will fail it the library cannot be found + from pylsl import pylsl + + # the find_liblsl_libraries() is a generator function that yields multiple possibilities + for libfile in pylsl.find_liblsl_libraries(): + if libfile: + break + except: + libfile = None + return libfile +# whenever a hook needs to load a 3rd party library, it needs to be done in an isolated subprocess +libfile = isolated.call(find_library) if libfile: # add the liblsl library to the binaries From dafc75b8e9994b22f66cdd8727b7a67288a279c8 Mon Sep 17 00:00:00 2001 From: Robert Oostenveld Date: Tue, 2 May 2023 10:01:48 +0200 Subject: [PATCH 07/13] point to the specific location for the brew recipe at labstreaminglayer/tap/lsl --- requirements-test-libraries.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-test-libraries.txt b/requirements-test-libraries.txt index 9a17d3a8..1dc07a77 100644 --- a/requirements-test-libraries.txt +++ b/requirements-test-libraries.txt @@ -139,7 +139,7 @@ pywin32-ctypes==0.2.0; sys_platform == 'win32' # pymediainfo on linux does not bundle mediainfo shared library, and requires system one. pymediainfo==6.0.1; sys_platform == 'darwin' or sys_platform == 'win32' -# the required library can be installed with "brew install lsl" on macOS, or with "conda install liblsl" on any platform +# the required library can be installed with "brew install labstreaminglayer/tap/lsl" on macOS, or with "conda install liblsl" on any platform pylsl==1.16.1; sys_platform == "darwin" # Include the requirements for testing From 80ce5f6d5586ffd94f0f2256dfc9b3d4ee7ee294 Mon Sep 17 00:00:00 2001 From: Robert Oostenveld Date: Tue, 2 May 2023 19:27:42 +0200 Subject: [PATCH 08/13] Deal with linter errors - src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py:14:53: W291 trailing whitespace - src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py:16:1: E302 expected 2 blank lines, found 1 - src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py:20:1: W293 blank line contains whitespace - src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py:25:5: E722 do not use bare 'except' - src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py:30:1: E305 expected 2 blank lines after class or function definition, found 1 --- .../hooks/stdhooks/hook-pylsl.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py b/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py index 6db30453..50143080 100644 --- a/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py +++ b/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py @@ -11,21 +11,24 @@ # ------------------------------------------------------------------ import os -from PyInstaller.utils.hooks import logger, isolated +from PyInstaller.utils.hooks import logger, isolated + def find_library(): try: # the import will fail it the library cannot be found from pylsl import pylsl - + # the find_liblsl_libraries() is a generator function that yields multiple possibilities for libfile in pylsl.find_liblsl_libraries(): if libfile: break - except: + except (ImportError, ModuleNotFoundError, RuntimeError) as error: + print(error) libfile = None return libfile + # whenever a hook needs to load a 3rd party library, it needs to be done in an isolated subprocess libfile = isolated.call(find_library) @@ -36,3 +39,4 @@ def find_library(): else: logger.warning("liblsl shared library not found - pylsl will likely fail to work!") binaries = [] + From 140e0d0603443a3aeee593fe2b282481d9f12138 Mon Sep 17 00:00:00 2001 From: Robert Oostenveld Date: Tue, 2 May 2023 23:15:52 +0200 Subject: [PATCH 09/13] also point to the correct lsl in the hidden files under .github --- .github/workflows/pr-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index a371bd17..7d975bc2 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -99,7 +99,7 @@ jobs: # Install libdiscid (dependency of discid python package). brew install libdiscid # Install lsl library for pylsl - brew install lsl + brew install brew install labstreaminglayer/tap/lsl - name: Install dependencies shell: bash From 0d888fc862d92d6259723761c3a9fe007ded1fbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Br=C3=A9nainn=20Woodsend?= Date: Tue, 2 May 2023 22:26:27 +0100 Subject: [PATCH 10/13] Fix brew command --- .github/workflows/pr-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 7d975bc2..b4a0687c 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -99,7 +99,7 @@ jobs: # Install libdiscid (dependency of discid python package). brew install libdiscid # Install lsl library for pylsl - brew install brew install labstreaminglayer/tap/lsl + brew install labstreaminglayer/tap/lsl - name: Install dependencies shell: bash From 5bad879e886317659b06cd36dfabad192a5a35ee Mon Sep 17 00:00:00 2001 From: Robert Oostenveld Date: Wed, 3 May 2023 09:36:28 +0200 Subject: [PATCH 11/13] added test for pylsl to test_libraries.py --- src/_pyinstaller_hooks_contrib/tests/test_libraries.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/_pyinstaller_hooks_contrib/tests/test_libraries.py b/src/_pyinstaller_hooks_contrib/tests/test_libraries.py index 72307da6..ae805328 100644 --- a/src/_pyinstaller_hooks_contrib/tests/test_libraries.py +++ b/src/_pyinstaller_hooks_contrib/tests/test_libraries.py @@ -251,6 +251,15 @@ def test_markdown(pyi_builder): """) +@importorskip('pylsl') +def test_lxml_isoschematron(pyi_builder): + pyi_builder.test_source( + """ + import pylsl + print(pylsl.version.__version__) + """) + + @importorskip('lxml') def test_lxml_isoschematron(pyi_builder): pyi_builder.test_source( From 7442f0713433edae8bcb5ec4703667d5ec80a699 Mon Sep 17 00:00:00 2001 From: Robert Oostenveld Date: Thu, 4 May 2023 10:44:06 +0200 Subject: [PATCH 12/13] fixed name of test function --- src/_pyinstaller_hooks_contrib/tests/test_libraries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_pyinstaller_hooks_contrib/tests/test_libraries.py b/src/_pyinstaller_hooks_contrib/tests/test_libraries.py index ae805328..a9607d1e 100644 --- a/src/_pyinstaller_hooks_contrib/tests/test_libraries.py +++ b/src/_pyinstaller_hooks_contrib/tests/test_libraries.py @@ -252,7 +252,7 @@ def test_markdown(pyi_builder): @importorskip('pylsl') -def test_lxml_isoschematron(pyi_builder): +def test_pylsl(pyi_builder): pyi_builder.test_source( """ import pylsl From 0e64cd15f389bc8a097880583b50227d5eb0d669 Mon Sep 17 00:00:00 2001 From: Robert Oostenveld Date: Thu, 4 May 2023 14:17:10 +0200 Subject: [PATCH 13/13] removed whitespace to address linter error "W293 blank line contains whitespace" --- src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py b/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py index 50143080..b235aa57 100644 --- a/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py +++ b/src/_pyinstaller_hooks_contrib/hooks/stdhooks/hook-pylsl.py @@ -39,4 +39,3 @@ def find_library(): else: logger.warning("liblsl shared library not found - pylsl will likely fail to work!") binaries = [] -