diff --git a/Changelog b/Changelog index cd89d8ce..fae25007 100644 --- a/Changelog +++ b/Changelog @@ -14,6 +14,17 @@ lot of work discussing OSX things with MB, and contributes fixes too. Releases ******** +* Unreleased + + * Libraries depending on ``@loader_path`` are now supported. + * ``@executable_path`` is supported and defaults to the current Python + executable path. This can be changed with the ``--executable-path`` flag. + * The ``--ignore-missing-dependencies`` flag can be given to ignore errors and + delocate as many dependencies as possible. + * Dependencies are now delocated recursively. + * ``delocate.delocating.copy_recurse`` has been deprecated. + * ``delocate.libsana.resolve_rpath`` has been deprecated. + * 0.8.2 (Sunday July 12th 2020) Bugfix release. diff --git a/delocate/cmd/delocate_path.py b/delocate/cmd/delocate_path.py index e24afb03..c69d9db7 100644 --- a/delocate/cmd/delocate_path.py +++ b/delocate/cmd/delocate_path.py @@ -22,7 +22,17 @@ def main(): Option("-d", "--dylibs-only", action="store_true", help="Only analyze files with known dynamic library " - "extensions")]) + "extensions"), + Option("--executable-path", + action="store", type='string', + default=os.path.dirname(sys.executable), + help="The path used to resolve @executable_path in dependencies" + ), + Option("--ignore-missing-dependencies", + action="store_true", + help="Skip dependencies which couldn't be found and delocate " + "as much as possible"), + ]) (opts, paths) = parser.parse_args() if len(paths) < 1: parser.print_help() @@ -37,7 +47,13 @@ def main(): print(path) # evaluate paths relative to the path we are working on lib_path = os.path.join(path, opts.lib_path) - delocate_path(path, lib_path, lib_filt_func) + delocate_path( + path, + lib_path, + lib_filt_func, + executable_path=opts.executable_path, + ignore_missing=opts.ignore_missing_dependencies, + ) if __name__ == '__main__': diff --git a/delocate/cmd/delocate_wheel.py b/delocate/cmd/delocate_wheel.py index f3ae1aa9..93efbedd 100644 --- a/delocate/cmd/delocate_wheel.py +++ b/delocate/cmd/delocate_wheel.py @@ -9,6 +9,8 @@ import os from os.path import join as pjoin, basename, exists, expanduser import sys +import logging +from typing import List, Optional, Text from optparse import OptionParser, Option @@ -16,6 +18,7 @@ def main(): + # type: () -> None parser = OptionParser( usage="%s WHEEL_FILENAME\n\n" % sys.argv[0] + __doc__, version="%prog " + __version__) @@ -28,8 +31,10 @@ def main(): help="Directory to store delocated wheels (default is to " "overwrite input)"), Option("-v", "--verbose", - action="store_true", - help="Show more verbose report of progress and failure"), + action="count", + help="Show a more verbose report of progress and failure." + " Additional flags show even more info, up to -vv.", + default=0), Option("-k", "--check-archs", action="store_true", help="Check architectures of depended libraries"), @@ -39,13 +44,28 @@ def main(): "extensions"), Option("--require-archs", action="store", type='string', - help=("Architectures that all wheel libraries should " - "have (from 'intel', 'i386', 'x86_64', 'i386,x86_64')" - "'universal2', 'x86_64,arm64'"))]) + help="Architectures that all wheel libraries should " + "have (from 'intel', 'i386', 'x86_64', 'i386,x86_64'" + "'universal2', 'x86_64,arm64')"), + Option("--executable-path", + action="store", type='string', + default=os.path.dirname(sys.executable), + help="The path used to resolve @executable_path in dependencies" + ), + Option("--ignore-missing-dependencies", + action="store_true", + help="Skip dependencies which couldn't be found and delocate " + "as much as possible"), + ]) (opts, wheels) = parser.parse_args() if len(wheels) < 1: parser.print_help() sys.exit(1) + logging.basicConfig( + level={ + 0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG + }.get(opts.verbose, logging.DEBUG) + ) multi = len(wheels) > 1 if opts.wheel_dir: wheel_dir = expanduser(opts.wheel_dir) @@ -53,6 +73,7 @@ def main(): os.makedirs(wheel_dir) else: wheel_dir = None + require_archs = None # type: Optional[List[Text]] if opts.require_archs is None: require_archs = [] if opts.check_archs else None elif ',' in opts.require_archs: @@ -67,10 +88,16 @@ def main(): out_wheel = pjoin(wheel_dir, basename(wheel)) else: out_wheel = wheel - copied = delocate_wheel(wheel, out_wheel, lib_filt_func=lib_filt_func, - lib_sdir=opts.lib_sdir, - require_archs=require_archs, - check_verbose=opts.verbose) + copied = delocate_wheel( + wheel, + out_wheel, + lib_filt_func=lib_filt_func, + lib_sdir=opts.lib_sdir, + require_archs=require_archs, + check_verbose=opts.verbose, + executable_path=opts.executable_path, + ignore_missing=opts.ignore_missing_dependencies, + ) if opts.verbose and len(copied): print("Copied to package {0} directory:".format(opts.lib_sdir)) copy_lines = [' ' + name for name in sorted(copied)] diff --git a/delocate/delocating.py b/delocate/delocating.py index 3d8b47ae..3390d251 100644 --- a/delocate/delocating.py +++ b/delocate/delocating.py @@ -6,17 +6,24 @@ import os from os.path import (join as pjoin, dirname, basename, exists, abspath, relpath, realpath) +import logging import shutil import warnings from subprocess import Popen, PIPE +from typing import (Callable, Dict, FrozenSet, Iterable, List, Mapping, + Optional, Set, Text, Tuple, Union) from .pycompat import string_types -from .libsana import tree_libs, stripped_lib_dict, get_rp_stripper +from .libsana import (tree_libs, stripped_lib_dict, get_rp_stripper, + walk_directory, get_dependencies) from .tools import (set_install_name, zip2dir, dir2zip, validate_signature, find_package_dirs, set_install_id, get_archs) from .tmpdirs import TemporaryDirectory from .wheeltools import rewrite_record, InWheel + +logger = logging.getLogger(__name__) + # Prefix for install_name_id of copied libraries DLC_PREFIX = '/DLC/' @@ -25,7 +32,12 @@ class DelocationError(Exception): pass -def delocate_tree_libs(lib_dict, lib_path, root_path): +def delocate_tree_libs( + lib_dict, # type: Mapping[Text, Mapping[Text, Text]] + lib_path, # type: Text + root_path # type: Text +): + # type: (...) -> Dict[Text, Dict[Text, Text]] """ Move needed libraries in `lib_dict` into `lib_path` `lib_dict` has keys naming libraries required by the files in the @@ -47,7 +59,7 @@ def delocate_tree_libs(lib_dict, lib_path, root_path): lib_path : str Path in which to store copies of libs referred to in keys of `lib_dict`. Assumed to exist - root_path : str, optional + root_path : str Root directory of tree analyzed in `lib_dict`. Any required library within the subtrees of `root_path` does not get copied, but libraries linking to it have links adjusted to use relative path to @@ -58,19 +70,51 @@ def delocate_tree_libs(lib_dict, lib_path, root_path): copied_libs : dict Filtered `lib_dict` dict containing only the (key, value) pairs from `lib_dict` where the keys are the libraries copied to `lib_path``. + + Raises + ------ + DelocationError + When a malformed `lib_dict` has unresolved paths, missing files, etc. + When two dependencies would've been be copied to the same destination. """ - copied_libs = {} - delocated_libs = set() + # Test for errors first to avoid getting half-way through changing the tree + libraries_to_copy, libraries_to_delocate = _analyze_tree_libs( + lib_dict, root_path + ) + # Copy libraries and update lib_dict. + lib_dict, copied_libraries = _copy_required_libs( + lib_dict, lib_path, root_path, libraries_to_copy + ) + # Update the install names of local and copied libaries. + _update_install_names( + lib_dict, root_path, libraries_to_delocate | copied_libraries + ) + return libraries_to_copy + + +def _analyze_tree_libs( + lib_dict, # type: Mapping[Text, Mapping[Text, Text]] + root_path # type: Text +): + # type: (...) -> Tuple[Dict[Text, Dict[Text, Text]], Set[Text]] + """ Verify then return which library files to copy and delocate. + + Returns + ------- + needs_copying : dict + The libraries outside of `root_path`. + This is in the `lib_dict` format for use by `delocate_tree_libs`. + needs_delocating : set of str + The libraries inside of `root_path` which need to be delocated. + """ + needs_delocating = set() # Libraries which need install names updated. + needs_copying = {} # A report of which libraries were copied. copied_basenames = set() rp_root_path = realpath(root_path) - rp_lib_path = realpath(lib_path) - # Test for errors first to avoid getting half-way through changing the tree for required, requirings in lib_dict.items(): - if required.startswith('@'): # assume @rpath etc are correct - # But warn, because likely they are not - warnings.warn('Not processing required path {0} because it ' - 'begins with @'.format(required)) - continue + if required.startswith('@'): + # @rpath, etc, at this point should never happen. + raise DelocationError("%s was expected to be resolved." % required) r_ed_base = basename(required) if relpath(required, rp_root_path).startswith('..'): # Not local, plan to copy @@ -80,29 +124,81 @@ def delocate_tree_libs(lib_dict, lib_path, root_path): if not exists(required): raise DelocationError('library "{0}" does not exist'.format( required)) - copied_libs[required] = requirings + # Copy requirings to preserve it since it will be modified later. + needs_copying[required] = dict(requirings) copied_basenames.add(r_ed_base) else: # Is local, plan to set relative loader_path - delocated_libs.add(required) - # Modify in place now that we've checked for errors - for required in copied_libs: - shutil.copy(required, lib_path) - # Set rpath and install names for this copied library - for requiring, orig_install_name in lib_dict[required].items(): - req_rel = relpath(rp_lib_path, dirname(requiring)) - set_install_name(requiring, orig_install_name, - '@loader_path/{0}/{1}'.format( - req_rel, basename(required))) - for required in delocated_libs: + needs_delocating.add(required) + return needs_copying, needs_delocating + + +def _copy_required_libs( + lib_dict, # type: Mapping[Text, Mapping[Text, Text]] + lib_path, # type: Text + root_path, # type: Text + libraries_to_copy # type: Iterable[Text] +): + # type (...) -> Tuple[Dict[Text, Dict[Text, Text]], Set[Text]] + """ Copy libraries outside of root_path to lib_path. + + Returns + ------- + updated_lib_dict : dict + A copy of `lib_dict` modified so that dependencies now point to + the copied library destinations. + needs_delocating : set of str + A set of the destination files, these need to be delocated. + """ + # Create a copy of lib_dict for this script to modify and return. + out_lib_dict = _copy_lib_dict(lib_dict) + del lib_dict + needs_delocating = set() # Set[Text] + for old_path in libraries_to_copy: + new_path = realpath(pjoin(lib_path, basename(old_path))) + logger.info( + "Copying library %s to %s", old_path, relpath(new_path, root_path) + ) + shutil.copy(old_path, new_path) + # Delocate this file now that it is stored locally. + needs_delocating.add(new_path) + # Update out_lib_dict with the new file paths. + out_lib_dict[new_path] = out_lib_dict[old_path] + del out_lib_dict[old_path] + for required in out_lib_dict: + if old_path not in out_lib_dict[required]: + continue + out_lib_dict[required][new_path] = out_lib_dict[required][old_path] + del out_lib_dict[required][old_path] + return out_lib_dict, needs_delocating + + +def _update_install_names( + lib_dict, # type: Mapping[Text, Mapping[Text, Text]] + root_path, # type: Text + files_to_delocate # type: Iterable[Text] +): + # type: (...) -> None + """ Update the install names of libraries. """ + for required in files_to_delocate: # Set relative path for local library for requiring, orig_install_name in lib_dict[required].items(): req_rel = relpath(required, dirname(requiring)) - set_install_name(requiring, orig_install_name, - '@loader_path/' + req_rel) - return copied_libs - - -def copy_recurse(lib_path, copy_filt_func=None, copied_libs=None): + new_install_name = '@loader_path/' + req_rel + logger.info( + "Modifying install name in %s from %s to %s", + relpath(requiring, root_path), + orig_install_name, + new_install_name, + ) + set_install_name(requiring, orig_install_name, new_install_name) + + +def copy_recurse( + lib_path, # type: Text + copy_filt_func=None, # type: Optional[Callable[[Text], bool]] + copied_libs=None # type: Optional[Dict[Text, Dict[Text, Text]]] +): + # type: (...) -> Dict[Text, Dict[Text, Text]] """ Analyze `lib_path` for library dependencies and copy libraries `lib_path` is a directory containing libraries. The libraries might @@ -135,7 +231,16 @@ def copy_recurse(lib_path, copy_filt_func=None, copied_libs=None): copied_libs : dict Input `copied_libs` dict with any extra libraries and / or dependencies added. + + .. deprecated:: 0.9 + This function is obsolete. :func:`delocate_path` handles recursive + dependencies while also supporting `@loader_path`. """ + warnings.warn( + "copy_recurse is obsolete and should no longer be called.", + DeprecationWarning, + stacklevel=2, + ) if copied_libs is None: copied_libs = {} else: @@ -148,8 +253,13 @@ def copy_recurse(lib_path, copy_filt_func=None, copied_libs=None): return copied_libs -def _copy_required(lib_path, copy_filt_func, copied_libs): - """ Copy libraries required for files in `lib_path` to `lib_path` +def _copy_required( + lib_path, # type: Text + copy_filt_func, # type: Optional[Callable[[Text], bool]] + copied_libs # type: Dict[Text, Dict[Text, Text]] +): + # type: (...) -> None + """ Copy libraries required for files in `lib_path` to `copied_libs` Augment `copied_libs` dictionary with any newly copied libraries, modifying `copied_libs` in-place - see Notes. @@ -197,6 +307,9 @@ def _copy_required(lib_path, copy_filt_func, copied_libs): ``/sys/libB.dylib``, so we fix our copy of `libC.dylib`` to point to ``my_lib_path/libB.dylib`` and add ``/sys/libC.dylib`` as a ``dependings_dict`` entry for ``copied_libs['/sys/libB.dylib']`` + + .. deprecated:: 0.9 + This function is obsolete, and is only used by :func:`copy_recurse`. """ # Paths will be prepended with `lib_path` lib_dict = tree_libs(lib_path) @@ -235,18 +348,26 @@ def _copy_required(lib_path, copy_filt_func, copied_libs): def _dylibs_only(filename): + # type: (Text) -> bool return (filename.endswith('.so') or filename.endswith('.dylib')) def filter_system_libs(libname): + # type: (Text) -> bool return not (libname.startswith('/usr/lib') or libname.startswith('/System')) -def delocate_path(tree_path, lib_path, - lib_filt_func=None, - copy_filt_func=filter_system_libs): +def delocate_path( + tree_path, # type: Text + lib_path, # type: Text + lib_filt_func=None, # type: Optional[Union[str, Callable[[Text], bool]]] + copy_filt_func=filter_system_libs, # type: Optional[Callable[[Text], bool]] + executable_path=None, # type: Optional[Text] + ignore_missing=False, # type: bool +): + # type: (...) -> Dict[Text, Dict[Text, Text]] """ Copy required libraries for files in `tree_path` into `lib_path` Parameters @@ -266,6 +387,11 @@ def delocate_path(tree_path, lib_path, Default is callable rejecting only libraries beginning with ``/usr/lib`` or ``/System``. None means copy all libraries. This will usually end up copying large parts of the system run-time. + executable_path : None or str, optional + If not None, an alternative path to use for resolving + `@executable_path`. + ignore_missing : bool, default=False + Continue even if missing dependencies are detected. Returns ------- @@ -277,38 +403,80 @@ def delocate_path(tree_path, lib_path, is a file in the path depending on ``copied_lib_path``, and the value is the ``install_name`` of ``copied_lib_path`` in the depending library. + + Raises + ------ + DelocationError + When any dependencies can not be located. """ if lib_filt_func == "dylibs-only": lib_filt_func = _dylibs_only + elif isinstance(lib_filt_func, str): + raise TypeError('lib_filt_func string can only be "dylibs-only"') + if lib_filt_func is None: + lib_filt_func = (lambda _: True) if not exists(lib_path): os.makedirs(lib_path) - lib_dict = tree_libs(tree_path, lib_filt_func) - if copy_filt_func is not None: - lib_dict = dict((key, value) for key, value in lib_dict.items() - if copy_filt_func(key)) - copied = delocate_tree_libs(lib_dict, lib_path, tree_path) - return copy_recurse(lib_path, copy_filt_func, copied) + + lib_dict = {} # type: Dict[Text, Dict[Text, Text]] + missing_libs = False + for library_path in walk_directory( + tree_path, lib_filt_func, executable_path=executable_path + ): + for depending_path, install_name in get_dependencies( + library_path, + executable_path=executable_path, + filt_func=lib_filt_func, + ): + if depending_path is None: + missing_libs = True + continue + if copy_filt_func and not copy_filt_func(depending_path): + continue + lib_dict.setdefault(depending_path, {}) + lib_dict[depending_path][library_path] = install_name + + if missing_libs and not ignore_missing: + # Details of missing libraries would have already reported by + # get_dependencies. + raise DelocationError("Could not find all dependencies.") + + return delocate_tree_libs(lib_dict, lib_path, tree_path) + + +def _copy_lib_dict(lib_dict): + # type: (Mapping[Text, Mapping[Text, Text]]) -> Dict[Text, Dict[Text, Text]] # noqa: E501 + """ Returns a copy of lib_dict. """ + return { # Convert nested Mapping types into nested Dict types. + required: dict(requiring) for required, requiring in lib_dict.items() + } def _merge_lib_dict(d1, d2): + # type: (Dict[Text, Dict[Text, Text]], Mapping[Text, Mapping[Text, Text]]) -> None # noqa: E501 """ Merges lib_dict `d2` into lib_dict `d1` """ for required, requirings in d2.items(): if required in d1: d1[required].update(requirings) else: - d1[required] = requirings + # Convert to dict to satisfy type checking. + d1[required] = dict(requirings) return None -def delocate_wheel(in_wheel, - out_wheel=None, - lib_sdir='.dylibs', - lib_filt_func=None, - copy_filt_func=filter_system_libs, - require_archs=None, - check_verbose=False, - ): +def delocate_wheel( + in_wheel, # type: Text + out_wheel=None, # type: Optional[Text] + lib_sdir='.dylibs', # type: Text + lib_filt_func=None, # type: Union[None, str, Callable[[Text], bool]] + copy_filt_func=filter_system_libs, # type: Optional[Callable[[Text], bool]] + require_archs=None, # type: Union[None, Text, Iterable[Text]] + check_verbose=False, # type: bool + executable_path=None, # type: Optional[Text] + ignore_missing=False, # type: bool +): + # type: (...) -> Dict[Text, Dict[Text, Text]] """ Update wheel by copying required libraries to `lib_sdir` in wheel Create `lib_sdir` in wheel tree only if we are copying one or more @@ -347,6 +515,10 @@ def delocate_wheel(in_wheel, (e.g "i386" or "x86_64"). check_verbose : bool, optional If True, print warning messages about missing required architectures + executable_path : str, optional + An alternative path to use for resolving `@executable_path`. + ignore_missing : bool, default=False + Continue even if missing dependencies are detected. Returns ------- @@ -359,8 +531,6 @@ def delocate_wheel(in_wheel, is the ``install_name`` of ``copied_lib_path`` in the depending library. The filenames in the keys are relative to the wheel root path. """ - if lib_filt_func == "dylibs-only": - lib_filt_func = _dylibs_only in_wheel = abspath(in_wheel) if out_wheel is None: out_wheel = in_wheel @@ -368,14 +538,20 @@ def delocate_wheel(in_wheel, out_wheel = abspath(out_wheel) in_place = in_wheel == out_wheel with TemporaryDirectory() as tmpdir: - all_copied = {} + all_copied = {} # type: Dict[Text, Dict[Text, Text]] wheel_dir = realpath(pjoin(tmpdir, 'wheel')) zip2dir(in_wheel, wheel_dir) for package_path in find_package_dirs(wheel_dir): lib_path = pjoin(package_path, lib_sdir) lib_path_exists = exists(lib_path) - copied_libs = delocate_path(package_path, lib_path, - lib_filt_func, copy_filt_func) + copied_libs = delocate_path( + package_path, + lib_path, + lib_filt_func, + copy_filt_func, + executable_path=executable_path, + ignore_missing=ignore_missing, + ) if copied_libs and lib_path_exists: raise DelocationError( '{0} already exists in wheel but need to copy ' @@ -409,6 +585,7 @@ def delocate_wheel(in_wheel, def patch_wheel(in_wheel, patch_fname, out_wheel=None): + # type: (Text, Text, Optional[Text]) -> None """ Apply ``-p1`` style patch in `patch_fname` to contents of `in_wheel` If `out_wheel` is None (the default), overwrite the wheel `in_wheel` @@ -450,7 +627,12 @@ def patch_wheel(in_wheel, patch_fname, out_wheel=None): } -def check_archs(copied_libs, require_archs=(), stop_fast=False): +def check_archs( + copied_libs, # type: Mapping[Text, Mapping[Text, Text]] + require_archs=(), # type: Union[Text, Iterable[Text]] + stop_fast=False # type: bool +): + # type: (...) -> Set[Union[Tuple[Text, FrozenSet[Text]], Tuple[Text, Text, FrozenSet[Text]]]] # noqa: E501 """ Check compatibility of archs in `copied_libs` dict Parameters @@ -492,17 +674,17 @@ def check_archs(copied_libs, require_archs=(), stop_fast=False): """ if isinstance(require_archs, string_types): require_archs = _ARCH_LOOKUP.get(require_archs, [require_archs]) - require_archs = frozenset(require_archs) - bads = [] + require_archs_set = frozenset(require_archs) + bads = [] # type: List[Union[Tuple[Text, FrozenSet[Text]], Tuple[Text, Text, FrozenSet[Text]]]] # noqa: E501 for depended_lib, dep_dict in copied_libs.items(): depended_archs = get_archs(depended_lib) for depending_lib, install_name in dep_dict.items(): depending_archs = get_archs(depending_lib) - all_required = depending_archs | require_archs + all_required = depending_archs | require_archs_set all_missing = all_required.difference(depended_archs) if len(all_missing) == 0: continue - required_missing = require_archs.difference(depended_archs) + required_missing = require_archs_set.difference(depended_archs) if len(required_missing): bads.append((depending_lib, required_missing)) else: diff --git a/delocate/libsana.py b/delocate/libsana.py index 5fdba8bc..9a99f2cc 100644 --- a/delocate/libsana.py +++ b/delocate/libsana.py @@ -4,16 +4,252 @@ """ import os -from os.path import basename, join as pjoin, realpath - +from os.path import basename, dirname, join as pjoin, realpath +import sys + +import logging +from typing import ( + Callable, + Dict, + Iterable, + Iterator, + List, + Optional, + Set, + Text, + Tuple, +) import warnings +import six + from .tools import (get_install_names, zip2dir, get_rpaths, get_environment_variable_paths) from .tmpdirs import TemporaryDirectory -def tree_libs(start_path, filt_func=None): +logger = logging.getLogger(__name__) + + +class DependencyNotFound(Exception): + """ + Raised by tree_libs or resolve_rpath if an expected dependency is missing. + """ + + +def _filter_system_libs(libname): + # type: (Text) -> bool + return not (libname.startswith('/usr/lib') or libname.startswith('/System')) + + +def get_dependencies( + lib_fname, # type: Text + executable_path=None, # type: Optional[Text] + filt_func=lambda filepath: True, # type: Callable[[str], bool] +): + # type: (...) -> Iterator[Tuple[Optional[Text], Text]] + """Find and yield the real paths of dependencies of the library `lib_fname` + + This function is used to search for the real files that are required by + `lib_fname`. + + The caller must check if any `dependency_path` is None and must decide on + how to handle missing dependencies. + + Parameters + ---------- + lib_fname : str + The library to fetch dependencies from. Must be an existing file. + executable_path : str, optional + An alternative path to use for resolving `@executable_path`. + filt_func : callable, optional + A callable which accepts filename as argument and returns True if we + should inspect the file or False otherwise. + Defaults to inspecting all files for library dependencies. + If `filt_func` returns False for `lib_fname` then no values will be + yielded. + If `filt_func` returns False for a dependencies real path then that + dependency will not be yielded. + + Yields + ------ + dependency_path : str or None + The real path of the dependencies of `lib_fname`. + If the library at `install_name` can not be found then this value will + be None. + install_name : str + The install name of `dependency_path` as if :func:`get_install_names` + was called. + + Raises + ------ + DependencyNotFound + When `lib_fname` does not exist. + """ + if not filt_func(lib_fname): + logger.debug("Ignoring dependencies of %s" % lib_fname) + return + if not os.path.isfile(lib_fname): + if not _filter_system_libs(lib_fname): + logger.debug( + "Ignoring missing library %s because it is a system library.", + lib_fname + ) + return + raise DependencyNotFound(lib_fname) + rpaths = get_rpaths(lib_fname) + get_environment_variable_paths() + for install_name in get_install_names(lib_fname): + try: + if install_name.startswith("@"): + dependency_path = resolve_dynamic_paths( + install_name, + rpaths, + loader_path=dirname(lib_fname), + executable_path=executable_path, + ) + else: + dependency_path = search_environment_for_lib(install_name) + if not os.path.isfile(dependency_path): + if not _filter_system_libs(dependency_path): + logger.debug( + "Skipped missing dependency %s" + " because it is a system library.", + dependency_path + ) + else: + raise DependencyNotFound(dependency_path) + if dependency_path != install_name: + logger.debug( + "%s resolved to: %s", install_name, dependency_path + ) + yield dependency_path, install_name + except DependencyNotFound: + message = "\n%s not found:\n Needed by: %s" % (install_name, + lib_fname) + if install_name.startswith("@rpath"): + message += "\n Search path:\n " + "\n ".join(rpaths) + logger.error(message) + # At this point install_name is known to be a bad path. + yield None, install_name + + +def walk_library( + lib_fname, # type: Text + filt_func=lambda filepath: True, # type: Callable[[Text], bool] + visited=None, # type: Optional[Set[Text]] + executable_path=None, # type: Optional[Text] +): + # type: (...) -> Iterator[Text] + """ + Yield all libraries on which `lib_fname` depends, directly or indirectly. + + First yields `lib_fname` itself, if not already `visited` and then all + dependencies of `lib_fname`, including dependencies of dependencies. + + Dependencies which can not be resolved will be logged and ignored. + + Parameters + ---------- + lib_fname : str + The library to start with. + filt_func : callable, optional + A callable which accepts filename as argument and returns True if we + should inspect the file or False otherwise. + Defaults to inspecting all files for library dependencies. + If `filt_func` filters a library it will also exclude all of that + libraries dependencies as well. + visited : None or set of str, optional + We update `visited` with new library_path's as we visit them, to + prevent infinite recursion and duplicates. Input value of None + corresponds to the set `{lib_path}`. Modified in-place. + executable_path : str, optional + An alternative path to use for resolving `@executable_path`. + + Yields + ------ + library_path : str + The path of each library depending on `lib_fname`, including + `lib_fname`, without duplicates. + """ + if visited is None: + visited = {lib_fname} + elif lib_fname in visited: + return + else: + visited.add(lib_fname) + if not filt_func(lib_fname): + logger.debug("Ignoring %s and its dependencies.", lib_fname) + return + yield lib_fname + for dependency_fname, install_name in get_dependencies( + lib_fname, executable_path=executable_path, filt_func=filt_func + ): + if dependency_fname is None: + logger.error( + "%s not found, requested by %s", install_name, lib_fname, + ) + continue + for sub_dependency in walk_library( + dependency_fname, + filt_func=filt_func, + visited=visited, + executable_path=executable_path, + ): + yield sub_dependency + + +def walk_directory( + root_path, # type: Text + filt_func=lambda filepath: True, # type: Callable[[Text], bool] + executable_path=None, # type: Optional[Text] +): + # type: (...) -> Iterator[Text] + """Walk along dependencies starting with the libraries within `root_path`. + + Dependencies which can not be resolved will be logged and ignored. + + Parameters + ---------- + root_path : str + The root directory to search for libraries depending on other libraries. + filt_func : None or callable, optional + A callable which accepts filename as argument and returns True if we + should inspect the file or False otherwise. + Defaults to inspecting all files for library dependencies. + If `filt_func` filters a library it will will not further analyze any + of that library's dependencies. + executable_path : None or str, optional + If not None, an alternative path to use for resolving + `@executable_path`. + + Yields + ------ + library_path : str + Iterates over the libraries in `root_path` and each of their + dependencies without any duplicates. + """ + visited_paths = set() # type: Set[Text] + for dirpath, dirnames, basenames in os.walk(root_path): + for base in basenames: + depending_path = realpath(pjoin(dirpath, base)) + if depending_path in visited_paths: + continue # A library in root_path was a dependency of another. + if not filt_func(depending_path): + continue + for library_path in walk_library( + depending_path, + filt_func=filt_func, + visited=visited_paths, + executable_path=executable_path + ): + yield library_path + + +def tree_libs( + start_path, # type: Text + filt_func=None, # type: Optional[Callable[[Text], bool]] +): + # type: (...) -> Dict[Text, Dict[Text, Text]] """ Return analysis of library dependencies within `start_path` Parameters @@ -24,16 +260,15 @@ def tree_libs(start_path, filt_func=None): If None, inspect all files for library dependencies. If callable, accepts filename as argument, returns True if we should inspect the file, False otherwise. - Returns ------- lib_dict : dict dictionary with (key, value) pairs of (``libpath``, ``dependings_dict``). - ``libpath`` is canonical (``os.path.realpath``) filename of library, or - library name starting with {'@rpath', '@loader_path', - '@executable_path'}. + ``libpath`` is a canonical (``os.path.realpath``) filename of library, + or library name starting with {'@loader_path'}. + ``dependings_dict`` is a dict with (key, value) pairs of (``depending_libpath``, ``install_name``), where ``dependings_libpath`` @@ -48,54 +283,126 @@ def tree_libs(start_path, filt_func=None): * https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man1/dyld.1.html # noqa: E501 * http://matthew-brett.github.io/pydagogue/mac_runtime_link.html + + .. deprecated:: 0.9 + This function does not support `@loader_path` and only returns the + direct dependencies of the libraries in `start_path`. """ - lib_dict = {} - env_var_paths = get_environment_variable_paths() + warnings.warn( + "tree_libs doesn't support @loader_path and has been deprecated.", + DeprecationWarning, + stacklevel=2, + ) + if filt_func is None: + filt_func = (lambda _: True) + lib_dict = {} # type: Dict[Text, Dict[Text, Text]] for dirpath, dirnames, basenames in os.walk(start_path): for base in basenames: - depending_libpath = realpath(pjoin(dirpath, base)) - if filt_func is not None and not filt_func(depending_libpath): - continue - rpaths = get_rpaths(depending_libpath) - search_paths = rpaths + env_var_paths - for install_name in get_install_names(depending_libpath): - # If the library starts with '@rpath' we'll try and resolve it - # We'll do nothing to other '@'-paths - # Otherwise we'll search for the library using env variables - if install_name.startswith('@rpath'): - lib_path = resolve_rpath(install_name, search_paths) - elif install_name.startswith('@'): - lib_path = install_name - else: - lib_path = search_environment_for_lib(install_name) - if lib_path in lib_dict: - lib_dict[lib_path][depending_libpath] = install_name - else: - lib_dict[lib_path] = {depending_libpath: install_name} + depending_path = realpath(pjoin(dirpath, base)) + for dependency_path, install_name in get_dependencies( + depending_path, filt_func=filt_func, + ): + if dependency_path is None: + # Mimic deprecated behavior. + # A lib_dict with unresolved paths is unsuitable for + # delocating, this is a missing dependency. + dependency_path = realpath(install_name) + if install_name.startswith("@loader_path/"): + # Support for `@loader_path` would break existing callers. + logger.debug( + "Excluding %s because it has '@loader_path'.", + install_name + ) + continue + lib_dict.setdefault(dependency_path, {}) + lib_dict[dependency_path][depending_path] = install_name return lib_dict +def resolve_dynamic_paths(lib_path, rpaths, loader_path, executable_path=None): + # type: (Text, Iterable[Text], Text, Optional[Text]) -> Text + """ Return `lib_path` with any special runtime linking names resolved. + + If `lib_path` has `@rpath` then returns the first `rpaths`/`lib_path` + combination found. If the library can't be found in `rpaths` then + DependencyNotFound is raised. + + `@loader_path` and `@executable_path` are resolved with their respective + parameters. + + Parameters + ---------- + lib_path : str + The path to a library file, which may or may not be a relative path + starting with `@rpath`, `@loader_path`, or `@executable_path`. + rpaths : sequence of str + A sequence of search paths, usually gotten from a call to `get_rpaths`. + loader_path : str + The path to be used for `@loader_path`. + This should be the directory of the library which is loading `lib_path`. + executable_path : None or str, optional + The path to be used for `@executable_path`. + If None is given then the path of the Python executable will be used. + + Returns + ------- + lib_path : str + A str with the resolved libraries realpath. + + Raises + ------ + DependencyNotFound + When `lib_path` has `@rpath` in it but no library can be found on any + of the provided `rpaths`. + """ + if executable_path is None: + executable_path = dirname(sys.executable) + if lib_path.startswith('@loader_path/'): + return realpath(pjoin(loader_path, lib_path.split('/', 1)[1])) + if lib_path.startswith('@executable_path/'): + return realpath(pjoin(executable_path, lib_path.split('/', 1)[1])) + if not lib_path.startswith('@rpath/'): + return realpath(lib_path) + + lib_rpath = lib_path.split('/', 1)[1] + for rpath in rpaths: + rpath_lib = resolve_dynamic_paths( + pjoin(rpath, lib_rpath), (), loader_path, executable_path + ) + if os.path.exists(rpath_lib): + return realpath(rpath_lib) + + raise DependencyNotFound(lib_path) + + def resolve_rpath(lib_path, rpaths): + # type: (Text, Iterable[Text]) -> Text """ Return `lib_path` with its `@rpath` resolved - If the `lib_path` doesn't have `@rpath` then it's returned as is. - If `lib_path` has `@rpath` then returns the first `rpaths`/`lib_path` combination found. If the library can't be found in `rpaths` then a detailed warning is printed and `lib_path` is returned as is. - Parameters ---------- lib_path : str The path to a library file, which may or may not start with `@rpath`. rpaths : sequence of str A sequence of search paths, usually gotten from a call to `get_rpaths`. - Returns ------- lib_path : str A str with the resolved libraries realpath. + + .. deprecated:: 0.9 + This function does not support `@loader_path`. + Use `resolve_dynamic_paths` instead. """ + warnings.warn( + "resolve_rpath doesn't support @loader_path and has been deprecated." + " Switch to using `resolve_dynamic_paths` instead.", + DeprecationWarning, + stacklevel=2, + ) if not lib_path.startswith('@rpath/'): return lib_path @@ -115,6 +422,7 @@ def resolve_rpath(lib_path, rpaths): def search_environment_for_lib(lib_path): + # type: (Text) -> Text """ Search common environment variables for `lib_path` We'll use a single approach here: @@ -166,6 +474,7 @@ def search_environment_for_lib(lib_path): def get_prefix_stripper(strip_prefix): + # type: (Text) -> Callable[[Text], Text] """ Return function to strip `strip_prefix` prefix from string if present Parameters @@ -182,11 +491,13 @@ def get_prefix_stripper(strip_prefix): n = len(strip_prefix) def stripper(path): + # type: (Text) -> Text return path if not path.startswith(strip_prefix) else path[n:] return stripper def get_rp_stripper(strip_path): + # type: (Text) -> Callable[[Text], Text] """ Return function to strip ``realpath`` of `strip_path` from string Parameters @@ -205,6 +516,7 @@ def get_rp_stripper(strip_path): def stripped_lib_dict(lib_dict, strip_prefix): + # type: (Dict[Text, Dict[Text, Text]], Text) -> Dict[Text, Dict[Text, Text]] """ Return `lib_dict` with `strip_prefix` removed from start of paths Use to give form of `lib_dict` that appears relative to some base path @@ -237,7 +549,11 @@ def stripped_lib_dict(lib_dict, strip_prefix): return relative_dict -def wheel_libs(wheel_fname, filt_func=None): +def wheel_libs( + wheel_fname, # type: Text + filt_func=None # type: Optional[Callable[[Text], bool]] +): + # type: (...) -> Dict[Text, Dict[Text, Text]] """ Return analysis of library dependencies with a Python wheel Use this routine for a dump of the dependency tree. @@ -268,7 +584,8 @@ def wheel_libs(wheel_fname, filt_func=None): def _paths_from_var(varname, lib_basename): - var = os.environ.get(varname) + # type: (Text, Text) -> List[Text] + var = os.environ.get(six.ensure_str(varname)) if var is None: return [] return [pjoin(path, lib_basename) for path in var.split(':')] diff --git a/delocate/tests/data/fakepkg1-1.0-cp36-abi3-macosx_10_9_universal2.whl b/delocate/tests/data/fakepkg1-1.0-cp36-abi3-macosx_10_9_universal2.whl index ed427bad..957c0f42 100644 Binary files a/delocate/tests/data/fakepkg1-1.0-cp36-abi3-macosx_10_9_universal2.whl and b/delocate/tests/data/fakepkg1-1.0-cp36-abi3-macosx_10_9_universal2.whl differ diff --git a/delocate/tests/data/fakepkg2-1.0-py3-none-any.whl b/delocate/tests/data/fakepkg2-1.0-py3-none-any.whl index 8b3d4e6c..97604888 100644 Binary files a/delocate/tests/data/fakepkg2-1.0-py3-none-any.whl and b/delocate/tests/data/fakepkg2-1.0-py3-none-any.whl differ diff --git a/delocate/tests/data/fakepkg_rpath-1.0-cp36-abi3-macosx_10_9_universal2.whl b/delocate/tests/data/fakepkg_rpath-1.0-cp36-abi3-macosx_10_9_universal2.whl index e041cbd1..8a3267b6 100644 Binary files a/delocate/tests/data/fakepkg_rpath-1.0-cp36-abi3-macosx_10_9_universal2.whl and b/delocate/tests/data/fakepkg_rpath-1.0-cp36-abi3-macosx_10_9_universal2.whl differ diff --git a/delocate/tests/data/libextfunc2_rpath.dylib b/delocate/tests/data/libextfunc2_rpath.dylib new file mode 100644 index 00000000..7336c14d Binary files /dev/null and b/delocate/tests/data/libextfunc2_rpath.dylib differ diff --git a/delocate/tests/data/libextfunc_rpath.dylib b/delocate/tests/data/libextfunc_rpath.dylib index 07c561f3..161bfaf1 100755 Binary files a/delocate/tests/data/libextfunc_rpath.dylib and b/delocate/tests/data/libextfunc_rpath.dylib differ diff --git a/delocate/tests/data/wheel_build_path.txt b/delocate/tests/data/wheel_build_path.txt index c262ec78..1f85f8bf 100644 --- a/delocate/tests/data/wheel_build_path.txt +++ b/delocate/tests/data/wheel_build_path.txt @@ -1 +1 @@ -/Users/brettmz-admin/dev_trees/delocate/wheel_makers +/Users/runner/work/delocate/delocate/wheel_makers diff --git a/delocate/tests/test_delocating.py b/delocate/tests/test_delocating.py index 9860078d..a93a4619 100644 --- a/delocate/tests/test_delocating.py +++ b/delocate/tests/test_delocating.py @@ -6,8 +6,12 @@ from os.path import (join as pjoin, dirname, basename, relpath, realpath, splitext) import shutil +from typing import Any, Iterable, List, Set, Text, Tuple from collections import namedtuple +import pytest +import six + from ..delocating import (DelocationError, delocate_tree_libs, copy_recurse, delocate_path, check_archs, bads_report, filter_system_libs) @@ -31,6 +35,7 @@ def _make_libtree(out_path): + # type: (Text) -> LibtreeLibs liba, libb, libc, test_lib = _copy_libs( [LIBA, LIBB, LIBC, TEST_LIB], out_path) sub_path = pjoin(out_path, 'subsub') @@ -70,7 +75,9 @@ def without_system_libs(obj): return out +@pytest.mark.filterwarnings("ignore:tree_libs:DeprecationWarning") def test_delocate_tree_libs(): + # type: () -> None # Test routine to copy library dependencies into a local directory with InTemporaryDirectory() as tmpdir: # Copy libs into a temporary directory @@ -175,10 +182,9 @@ def test_delocate_tree_libs(): ext_local_libs = {liba, libb, libc, slibc} assert_equal(set(os.listdir(copy_dir2)), set([basename(lib) for lib in ext_local_libs])) - # Libraries using the copied libraries now have an install name starting # noqa: E501 + # Libraries using the copied libraries now have an install name starting # with @loader_path, then pointing to the copied library directory - all_local_libs = liba, libb, libc, test_lib, slibc, stest_lib - for lib in all_local_libs: + for lib in (liba, libb, libc, test_lib, slibc, stest_lib): pathto_copies = relpath(realpath(copy_dir2), dirname(realpath(lib))) lib_inames = get_install_names(lib) @@ -189,6 +195,7 @@ def test_delocate_tree_libs(): def _copy_fixpath(files, directory): + # type: (Iterable[Text], Text) -> List[Text] new_fnames = [] for fname in files: shutil.copy2(fname, directory) @@ -201,12 +208,16 @@ def _copy_fixpath(files, directory): def _copy_to(fname, directory, new_base): + # type: (Text, Text, Text) -> Text new_name = pjoin(directory, new_base) shutil.copy2(fname, new_name) return new_name +@pytest.mark.filterwarnings("ignore:tree_libs:DeprecationWarning") +@pytest.mark.filterwarnings("ignore:copy_recurse:DeprecationWarning") def test_copy_recurse(): + # type: () -> None # Function to find / copy needed libraries recursively with InTemporaryDirectory(): # Get some fixed up libraries to play with @@ -220,6 +231,7 @@ def test_copy_recurse(): # One library, depends only on system libs, system libs filtered def filt_func(libname): + # type: (Text) -> bool return not libname.startswith('/usr/lib') os.makedirs('subtree') _copy_fixpath([LIBA], 'subtree') @@ -233,6 +245,7 @@ def filt_func(libname): # copied first, then liba, libb def _st(fname): + # type: (Text) -> Text return _rp(pjoin('subtree2', basename(fname))) os.makedirs('subtree2') shutil.copy2(test_lib, 'subtree2') @@ -304,7 +317,10 @@ def _st(fname): assert_equal(copied_libs, copied_copied) +@pytest.mark.filterwarnings("ignore:tree_libs:DeprecationWarning") +@pytest.mark.filterwarnings("ignore:copy_recurse:DeprecationWarning") def test_copy_recurse_overwrite(): + # type: () -> None # Check that copy_recurse won't overwrite pre-existing libs with InTemporaryDirectory(): # Get some fixed up libraries to play with @@ -314,6 +330,7 @@ def test_copy_recurse_overwrite(): # Filter system libs def filt_func(libname): + # type: (Text) -> bool return not libname.startswith('/usr/lib') os.makedirs('subtree') # libb depends on liba @@ -327,6 +344,7 @@ def filt_func(libname): def test_delocate_path(): + # type: () -> None # Test high-level path delocator script with InTemporaryDirectory(): # Make a tree; use realpath for OSX /private/var - /var @@ -358,6 +376,7 @@ def test_delocate_path(): set_install_name(slibc, EXT_LIBS[0], fake_lib) def filt(libname): + # type: (Text) -> bool return not (libname.startswith('/usr') or 'libfake' in libname) assert_equal(delocate_path('subtree3', 'deplibs3', None, filt), {}) @@ -368,6 +387,7 @@ def filt(libname): set_install_name(slibc, EXT_LIBS[0], fake_lib) def lib_filt(filename): + # type: (Text) -> bool return not filename.endswith('subsub/libc.dylib') assert_equal(delocate_path('subtree4', 'deplibs4', lib_filt), {}) assert_equal(len(os.listdir('deplibs4')), 0) @@ -380,6 +400,7 @@ def lib_filt(filename): def _make_bare_depends(): + # type: () -> Tuple[Text, Text] # Copy: # * liba.dylib to 'libs' dir, which is a dependency of libb.dylib # * libb.dylib to 'subtree' dir, as 'libb' (no extension). @@ -396,6 +417,7 @@ def _make_bare_depends(): def test_delocate_path_dylibs(): + # type: () -> None # Test options for delocating everything, or just dynamic libraries _rp = realpath # shortcut with InTemporaryDirectory(): @@ -414,10 +436,12 @@ def test_delocate_path_dylibs(): liba, bare_b = _make_bare_depends() def func(fn): + # type: (Text) -> bool return fn.endswith('.dylib') assert_equal(delocate_path('subtree', 'deplibs', func), {}) def func(fn): + # type: (Text) -> bool return fn.endswith('libb') assert_equal(delocate_path('subtree', 'deplibs', None), {_rp(pjoin('libs', 'liba.dylib')): @@ -425,9 +449,10 @@ def func(fn): def test_check_archs(): + # type: () -> None # Test utility to check architectures in copied_libs dict # No libs always OK - s0 = set() + s0 = set() # type: Set[Any] assert_equal(check_archs({}), s0) # One lib to itself OK lib_M1_M1 = {LIBM1: {LIBM1: 'install_name'}} @@ -435,7 +460,7 @@ def test_check_archs(): assert_equal(check_archs(lib_M1_M1), s0) assert_equal(check_archs(lib_64_64), s0) # OK matching to another static lib of same arch - assert_equal(check_archs({LIB64A: {LIB64: 'install_name'}}), s0) + assert_equal(check_archs({LIB64A: {LIB64: u'install_name'}}), s0) # Or two libs two_libs = {LIB64A: {LIB64: 'install_name'}, LIBM1: {LIBM1: 'install_name'}} @@ -505,6 +530,7 @@ def test_check_archs(): def test_bads_report(): + # type: () -> None # Test bads_report of architecture errors # No bads, no report assert_equal(bads_report(set()), '') @@ -543,6 +569,7 @@ def test_bads_report(): def test_dyld_library_path_lookups(): + # type: () -> None # Test that DYLD_LIBRARY_PATH can be used to find libs during # delocation with TempDirWithoutEnvVars('DYLD_LIBRARY_PATH') as tmpdir: @@ -565,6 +592,7 @@ def test_dyld_library_path_lookups(): def test_dyld_library_path_beats_basename(): + # type: () -> None # Test that we find libraries on DYLD_LIBRARY_PATH before basename with TempDirWithoutEnvVars('DYLD_LIBRARY_PATH') as tmpdir: # Copy libs into a temporary directory @@ -582,12 +610,13 @@ def test_dyld_library_path_beats_basename(): # /private/var, so we'll use realpath to resolve the two assert_equal(predicted_lib_location, os.path.realpath(libb)) # Updating shows us the new lib - os.environ['DYLD_LIBRARY_PATH'] = subdir + os.environ['DYLD_LIBRARY_PATH'] = six.ensure_str(subdir) predicted_lib_location = search_environment_for_lib(libb) assert_equal(predicted_lib_location, new_libb) def test_dyld_fallback_library_path_loses_to_basename(): + # type: () -> None # Test that we find libraries on basename before DYLD_FALLBACK_LIBRARY_PATH with TempDirWithoutEnvVars('DYLD_FALLBACK_LIBRARY_PATH') as tmpdir: # Copy libs into a temporary directory diff --git a/delocate/tests/test_libsana.py b/delocate/tests/test_libsana.py index 8f70ab35..135be071 100644 --- a/delocate/tests/test_libsana.py +++ b/delocate/tests/test_libsana.py @@ -5,9 +5,17 @@ import os from os.path import (join as pjoin, dirname, realpath, relpath, split) +import shutil +from typing import Dict, Iterable, Text +import pytest + +from ..delocating import filter_system_libs from ..libsana import (tree_libs, get_prefix_stripper, get_rp_stripper, - stripped_lib_dict, wheel_libs, resolve_rpath) + stripped_lib_dict, wheel_libs, resolve_rpath, + get_dependencies, resolve_dynamic_paths, + walk_library, walk_directory, + DependencyNotFound) from ..tools import set_install_name @@ -16,11 +24,12 @@ from .pytest_tools import assert_equal from .test_install_names import (LIBA, LIBB, LIBC, TEST_LIB, _copy_libs, - EXT_LIBS, LIBSYSTEMB) + EXT_LIBS, LIBSYSTEMB, DATA_PATH) from .test_wheelies import (PLAT_WHEEL, PURE_WHEEL, STRAY_LIB_DEP) def get_ext_dict(local_libs): + # type: (Iterable[Text]) -> Dict[Text, Dict[Text, Text]] ext_deps = {} for ext_lib in EXT_LIBS: lib_deps = {} @@ -30,7 +39,9 @@ def get_ext_dict(local_libs): return ext_deps +@pytest.mark.filterwarnings("ignore:tree_libs:DeprecationWarning") def test_tree_libs(): + # type: () -> None # Test ability to walk through tree, finding dynamic libary refs # Copy specific files to avoid working tree cruft to_copy = [LIBA, LIBB, LIBC, TEST_LIB] @@ -45,16 +56,17 @@ def test_tree_libs(): rp_libb: {rp_libc: 'libb.dylib'}, rp_libc: {rp_test_lib: 'libc.dylib'}}) # default - no filtering - assert_equal(tree_libs(tmpdir), exp_dict) + assert tree_libs(tmpdir) == exp_dict def filt(fname): + # type: (Text) -> bool return fname.endswith('.dylib') exp_dict = get_ext_dict([liba, libb, libc]) exp_dict.update({ rp_liba: {rp_libb: 'liba.dylib', rp_libc: 'liba.dylib'}, rp_libb: {rp_libc: 'libb.dylib'}}) # filtering - assert_equal(tree_libs(tmpdir, filt), exp_dict) + assert tree_libs(tmpdir, filt) == exp_dict # Copy some libraries into subtree to test tree walking subtree = pjoin(tmpdir, 'subtree') slibc, stest_lib = _copy_libs([libc, test_lib], subtree) @@ -65,7 +77,7 @@ def filt(fname): realpath(slibc): 'liba.dylib'}, rp_libb: {rp_libc: 'libb.dylib', realpath(slibc): 'libb.dylib'}}) - assert_equal(tree_libs(tmpdir, filt), st_exp_dict) + assert tree_libs(tmpdir, filt) == st_exp_dict # Change an install name, check this is picked up set_install_name(slibc, 'liba.dylib', 'newlib') inc_exp_dict = get_ext_dict([liba, libb, libc, slibc]) @@ -75,11 +87,11 @@ def filt(fname): realpath('newlib'): {realpath(slibc): 'newlib'}, rp_libb: {rp_libc: 'libb.dylib', realpath(slibc): 'libb.dylib'}}) - assert_equal(tree_libs(tmpdir, filt), inc_exp_dict) + assert tree_libs(tmpdir, filt) == inc_exp_dict # Symlink a depending canonical lib - should have no effect because of # the canonical names os.symlink(liba, pjoin(dirname(liba), 'funny.dylib')) - assert_equal(tree_libs(tmpdir, filt), inc_exp_dict) + assert tree_libs(tmpdir, filt) == inc_exp_dict # Symlink a depended lib. Now 'newlib' is a symlink to liba, and the # dependency of slibc on newlib appears as a dependency on liba, but # with install name 'newlib' @@ -91,10 +103,11 @@ def filt(fname): realpath(slibc): 'newlib'}, rp_libb: {rp_libc: 'libb.dylib', realpath(slibc): 'libb.dylib'}}) - assert_equal(tree_libs(tmpdir, filt), sl_exp_dict) + assert tree_libs(tmpdir, filt) == sl_exp_dict def test_get_prefix_stripper(): + # type: () -> None # Test function factory to strip prefixes f = get_prefix_stripper('') assert_equal(f('a string'), 'a string') @@ -105,6 +118,7 @@ def test_get_prefix_stripper(): def test_get_rp_stripper(): + # type: () -> None # realpath prefix stripper # Just does realpath and adds path sep cwd = realpath(os.getcwd()) @@ -118,6 +132,7 @@ def test_get_rp_stripper(): def get_ext_dict_stripped(local_libs, start_path): + # type: (Iterable[Text], Text) -> Dict[Text, Dict[Text, Text]] ext_dict = {} for ext_lib in EXT_LIBS: lib_deps = {} @@ -131,6 +146,7 @@ def get_ext_dict_stripped(local_libs, start_path): def test_stripped_lib_dict(): + # type: () -> None # Test routine to return lib_dict with relative paths to_copy = [LIBA, LIBB, LIBC, TEST_LIB] with InTemporaryDirectory() as tmpdir: @@ -164,6 +180,7 @@ def test_stripped_lib_dict(): def test_wheel_libs(): + # type: () -> None # Test routine to list dependencies from wheels assert_equal(wheel_libs(PURE_WHEEL), {}) mod2 = pjoin('fakepkg1', 'subpkg', 'module2.abi3.so') @@ -173,11 +190,28 @@ def test_wheel_libs(): realpath(LIBSYSTEMB): {mod2: LIBSYSTEMB}}) def filt(fname): + # type: (Text) -> bool return not fname.endswith(mod2) assert wheel_libs(PLAT_WHEEL, filt) == {} +def test_resolve_dynamic_paths(): + # type: () -> None + # A minimal test of the resolve_rpath function + path, lib = split(LIBA) + lib_rpath = pjoin('@rpath', lib) + # Should skip '/nonexist' path + assert ( + resolve_dynamic_paths(lib_rpath, ['/nonexist', path], path) + == realpath(LIBA) + ) + # Should raise DependencyNotFound if the dependency can not be resolved. + with pytest.raises(DependencyNotFound): + resolve_dynamic_paths(lib_rpath, [], path) + + def test_resolve_rpath(): + # type: () -> None # A minimal test of the resolve_rpath function path, lib = split(LIBA) lib_rpath = pjoin('@rpath', lib) @@ -185,3 +219,62 @@ def test_resolve_rpath(): assert_equal(resolve_rpath(lib_rpath, ['/nonexist', path]), realpath(LIBA)) # Should return the given parameter as is since it can't be found assert_equal(resolve_rpath(lib_rpath, []), lib_rpath) + + +def test_get_dependencies(tmpdir): + # type: (object) -> None + tmpdir = str(tmpdir) + with pytest.raises(DependencyNotFound): + list(get_dependencies("nonexistent.lib")) + ext_libs = {(lib, lib) for lib in EXT_LIBS} + assert set(get_dependencies(LIBA)) == ext_libs + + os.symlink( + pjoin(DATA_PATH, "libextfunc_rpath.dylib"), + pjoin(tmpdir, "libextfunc_rpath.dylib"), + ) + assert set(get_dependencies(pjoin(tmpdir, "libextfunc_rpath.dylib"), + filt_func=filter_system_libs)) == { + (None, "@rpath/libextfunc2_rpath.dylib"), (LIBSYSTEMB, LIBSYSTEMB), + } + + assert set(get_dependencies(pjoin(tmpdir, "libextfunc_rpath.dylib"), + executable_path=DATA_PATH, + filt_func=filter_system_libs)) == { + (pjoin(DATA_PATH, "libextfunc2_rpath.dylib"), + "@rpath/libextfunc2_rpath.dylib" + ), + (LIBSYSTEMB, LIBSYSTEMB), + } + + +def test_walk_library(): + # type: () -> None + with pytest.raises(DependencyNotFound): + list(walk_library("nonexistent.lib")) + assert set(walk_library(LIBA, filt_func=filter_system_libs)) == {LIBA, } + assert set(walk_library(pjoin(DATA_PATH, "libextfunc_rpath.dylib"), + filt_func=filter_system_libs)) == { + pjoin(DATA_PATH, "libextfunc_rpath.dylib"), + pjoin(DATA_PATH, "libextfunc2_rpath.dylib"), + } + + +def test_walk_directory(tmpdir): + # type: (object) -> None + tmpdir = str(tmpdir) + assert set(walk_directory(tmpdir)) == set() + + shutil.copy(pjoin(DATA_PATH, "libextfunc_rpath.dylib"), tmpdir) + assert set(walk_directory(tmpdir, filt_func=filter_system_libs)) == { + pjoin(tmpdir, "libextfunc_rpath.dylib"), + } + + assert set( + walk_directory( + tmpdir, executable_path=DATA_PATH, filt_func=filter_system_libs + ) + ) == { + pjoin(tmpdir, "libextfunc_rpath.dylib"), + pjoin(DATA_PATH, "libextfunc2_rpath.dylib"), + } diff --git a/delocate/tests/test_scripts.py b/delocate/tests/test_scripts.py index fc9eaf40..ee57a023 100644 --- a/delocate/tests/test_scripts.py +++ b/delocate/tests/test_scripts.py @@ -10,8 +10,9 @@ import os from os.path import (dirname, join as pjoin, isfile, abspath, realpath, - basename, exists, splitext) + basename, exists, splitext, sep as psep) import shutil +from typing import Text from ..tmpdirs import InTemporaryDirectory from ..tools import back_tick, set_install_name, zip2dir, dir2zip @@ -61,10 +62,20 @@ def _proc_lines(in_str): def test_listdeps(): # smokey tests of list dependencies command - local_libs = set(['liba.dylib', 'libb.dylib', 'libc.dylib']) + libext_rpath = realpath(pjoin(DATA_PATH, 'libextfunc2_rpath.dylib')) + rp_cwd = realpath(os.getcwd()) + psep + # Replicate path stripping. + if libext_rpath.startswith(rp_cwd): + libext_rpath = libext_rpath[len(rp_cwd):] + local_libs = { + 'liba.dylib', + 'libb.dylib', + 'libc.dylib', + libext_rpath, + } # single path, with libs code, stdout, stderr = run_command(['delocate-listdeps', DATA_PATH]) - assert_equal(set(stdout), local_libs) + assert set(stdout) == local_libs assert_equal(code, 0) # single path, no libs with InTemporaryDirectory(): @@ -230,6 +241,7 @@ def test_fix_wheel_dylibs(): def test_fix_wheel_archs(): + # type: () -> None # Some tests for wheel fixing with InTemporaryDirectory() as tmpdir: # Test check of architectures @@ -244,10 +256,12 @@ def test_fix_wheel_archs(): archs = set(('x86_64', 'arm64')) def _fix_break(arch): + # type: (Text) -> None _fixed_wheel(tmpdir) _thin_lib(stray_lib, arch) def _fix_break_fix(arch): + # type: (Text) -> None _fixed_wheel(tmpdir) _thin_lib(stray_lib, arch) _thin_mod(fixed_wheel, arch) @@ -264,10 +278,10 @@ def _fix_break_fix(arch): ['delocate-wheel', fixed_wheel, '--check-archs'], check_code=False) assert_false(code == 0) - stderr = stderr.decode('latin1').strip() - assert_true(stderr.startswith('Traceback')) - assert_true(stderr.endswith( - "Some missing architectures in wheel")) + stderr_unicode = stderr.decode('latin1').strip() + assert stderr_unicode.startswith('Traceback') + assert stderr_unicode.endswith( + "Some missing architectures in wheel") assert_equal(stdout.strip(), b'') # Checked, verbose _fix_break(arch) @@ -276,11 +290,11 @@ def _fix_break_fix(arch): check_code=False) assert_false(code == 0) stderr = stderr.decode('latin1').strip() - assert_true(stderr.startswith('Traceback')) + assert 'Traceback' in stderr assert_true(stderr.endswith( "Some missing architectures in wheel")) - stdout = stdout.decode('latin1').strip() - assert_equal(stdout, + stdout_unicode = stdout.decode('latin1').strip() + assert_equal(stdout_unicode, fmt_str.format( fixed_wheel, 'fakepkg1/subpkg/module2.abi3.so', diff --git a/delocate/tests/test_wheelies.py b/delocate/tests/test_wheelies.py index d4daf6a7..fc8a13ac 100644 --- a/delocate/tests/test_wheelies.py +++ b/delocate/tests/test_wheelies.py @@ -302,17 +302,23 @@ def test_fix_rpath(): # The module was set to expect its dependency in the libs/ directory os.symlink(DATA_PATH, 'libs') - stray_lib = realpath('libs/libextfunc_rpath.dylib') with InWheel(RPATH_WHEEL): # dep_mod can vary depending the Python version used to build # the test wheel dep_mod = 'fakepkg/subpkg/module2.abi3.so' dep_path = '@rpath/libextfunc_rpath.dylib' - assert_equal( - delocate_wheel(RPATH_WHEEL, 'tmp.whl'), - {stray_lib: {dep_mod: dep_path}}, - ) + stray_libs = { + realpath('libs/libextfunc_rpath.dylib'): {dep_mod: dep_path}, + realpath('libs/libextfunc2_rpath.dylib'): { + realpath( + 'libs/libextfunc_rpath.dylib' + ): '@rpath/libextfunc2_rpath.dylib' + }, + } + + assert delocate_wheel(RPATH_WHEEL, 'tmp.whl') == stray_libs + with InWheel('tmp.whl'): check_call(['codesign', '--verify', 'fakepkg/.dylibs/libextfunc_rpath.dylib']) diff --git a/setup.py b/setup.py index eba22975..7a18bf83 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,9 @@ "machomachomangler; sys_platform == 'win32'", "bindepend; sys_platform == 'win32'", "wheel", + "six", + "typing; python_version < '3.5'", + "typing_extensions", ], package_data={'delocate.tests': [pjoin('data', '*.dylib'), diff --git a/wheel_makers/fakepkg_rpath/libs/extfunc.c b/wheel_makers/fakepkg_rpath/libs/extfunc.c index 2de5e72f..8c9dbd0b 100644 --- a/wheel_makers/fakepkg_rpath/libs/extfunc.c +++ b/wheel_makers/fakepkg_rpath/libs/extfunc.c @@ -1,4 +1,6 @@ +int extfunc2(); + int extfunc() { - return 3; + return extfunc2(); } diff --git a/wheel_makers/fakepkg_rpath/libs/extfunc2.c b/wheel_makers/fakepkg_rpath/libs/extfunc2.c new file mode 100644 index 00000000..e1e4287f --- /dev/null +++ b/wheel_makers/fakepkg_rpath/libs/extfunc2.c @@ -0,0 +1,4 @@ +int extfunc2() +{ + return 3; +} diff --git a/wheel_makers/fakepkg_rpath/setup.py b/wheel_makers/fakepkg_rpath/setup.py index f7549435..ed8cfdf9 100644 --- a/wheel_makers/fakepkg_rpath/setup.py +++ b/wheel_makers/fakepkg_rpath/setup.py @@ -9,17 +9,33 @@ HERE = abspath(dirname(__file__)) LIBS = pjoin(HERE, 'libs') -EXTLIB = pjoin(LIBS, 'libextfunc_rpath.dylib') -INSTALL_NAME = '@rpath/libextfunc_rpath.dylib' arch_flags = ['-arch', 'arm64', '-arch', 'x86_64'] # dual arch + +EXTLIB2 = pjoin(LIBS, 'libextfunc2_rpath.dylib') +INSTALL_NAME2 = '@rpath/libextfunc2_rpath.dylib' +check_call([ + 'cc', pjoin(LIBS, 'extfunc2.c'), + '-dynamiclib', + '-install_name', INSTALL_NAME2, + '-o', EXTLIB2, +] + arch_flags) + +EXTLIB = pjoin(LIBS, 'libextfunc_rpath.dylib') +INSTALL_NAME = '@rpath/libextfunc_rpath.dylib' check_call([ 'cc', pjoin(LIBS, 'extfunc.c'), '-dynamiclib', '-install_name', INSTALL_NAME, + '-L', LIBS, + '-l', 'extfunc2_rpath', + '-rpath', "@executable_path/", + '-rpath', "@loader_path/", '-o', EXTLIB, ] + arch_flags) + check_call(['codesign', '--sign', '-', EXTLIB]) +check_call(['codesign', '--sign', '-', EXTLIB2]) exts = [ Extension(