From 16367240bb95c65b24152539fefa5095404f03f9 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Mon, 22 Apr 2024 17:03:46 +0000 Subject: [PATCH 1/2] `get_software_libdir`: return dir if it's only 1 with libs There are situations e.g. with recent Gentoo where all 64-bit libraries are under `lib64`, but `lib` is seperate and used for other files and directories (e.g. `lib/python`), and for 32-bit libraries. `get_software_libdir` would fail for such. To avoid this: if `only_one` is `True` (default), *and* no specific file (`fs`) is specified (also default), *and* `lib` and `lib64` both exist and are separate, and only one of them has libraries (shared or static), it returns that one directory. --- easybuild/tools/modules.py | 10 ++++++++++ test/framework/modules.py | 7 ++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index fc21a564b9..a24ddaa369 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -37,6 +37,7 @@ * Jens Timmerman (Ghent University) * David Brown (Pacific Northwest National Laboratory) """ +import glob import os import re import shlex @@ -52,6 +53,7 @@ from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX from easybuild.tools.py2vs3 import subprocess_popen_text from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.utilities import get_subclasses, nub # software root/version environment variable name prefixes @@ -1671,6 +1673,7 @@ def get_software_libdir(name, only_one=True, fs=None): Returns the library subdirectory, relative to software root. It fails if multiple library subdirs are found, unless only_one is False which yields a list of all library subdirs. + If only_one is True and fs is None, select the one subdirectory with shared or static libraries, if possible. :param name: name of the software package :param only_one: indicates whether only one lib path is expected to be found @@ -1703,6 +1706,13 @@ def get_software_libdir(name, only_one=True, fs=None): if len(res) == 1: res = res[0] else: + if fs is None: + # check if only one (exactly) has libraries + # this is needed for software with library archives in lib64 but other files/directories in lib + lib_glob = ["*.%s" % ext for ext in ["a", get_shared_lib_ext()]] + haslibs = [any(glob.glob(os.path.join(root, subdir, f)) for f in lib_glob) for subdir in res] + if haslibs[0] != haslibs[1]: + return res[int(not haslibs[0])] raise EasyBuildError("Multiple library subdirectories found for %s in %s: %s", name, root, ', '.join(res)) return res diff --git a/test/framework/modules.py b/test/framework/modules.py index 82d03f6440..bb11802db7 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -674,10 +674,15 @@ def test_get_software_root_version_libdir(self): os.environ.pop('EBROOT%s' % env_var_name) os.environ.pop('EBVERSION%s' % env_var_name) - # check expected result of get_software_libdir with multiple lib subdirs + # if only 'lib' has a library archive, use it root = os.path.join(tmpdir, name) mkdir(os.path.join(root, 'lib64')) os.environ['EBROOT%s' % env_var_name] = root + write_file(os.path.join(root, 'lib', 'foo.a'), 'foo') + self.assertEqual(get_software_libdir(name), 'lib') + + # check expected result of get_software_libdir with multiple lib subdirs + remove_file(os.path.join(root, 'lib', 'foo.a')) self.assertErrorRegex(EasyBuildError, "Multiple library subdirectories found.*", get_software_libdir, name) self.assertEqual(get_software_libdir(name, only_one=False), ['lib', 'lib64']) From 171fdb81465d362fad0d223981edcb914c268fb2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 7 Jun 2024 17:07:44 +0200 Subject: [PATCH 2/2] code style cleanup in get_software_libdir + enhance test --- easybuild/tools/modules.py | 15 +++++++++------ test/framework/modules.py | 13 +++++++++++-- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index a24ddaa369..7318be4f4f 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1706,13 +1706,16 @@ def get_software_libdir(name, only_one=True, fs=None): if len(res) == 1: res = res[0] else: - if fs is None: - # check if only one (exactly) has libraries + if fs is None and len(res) == 2: + # if both lib and lib64 were found, check if only one (exactly) has libraries; # this is needed for software with library archives in lib64 but other files/directories in lib - lib_glob = ["*.%s" % ext for ext in ["a", get_shared_lib_ext()]] - haslibs = [any(glob.glob(os.path.join(root, subdir, f)) for f in lib_glob) for subdir in res] - if haslibs[0] != haslibs[1]: - return res[int(not haslibs[0])] + lib_glob = ['*.%s' % ext for ext in ['a', get_shared_lib_ext()]] + has_libs = [any(glob.glob(os.path.join(root, subdir, f)) for f in lib_glob) for subdir in res] + if has_libs[0] and not has_libs[1]: + return res[0] + elif has_libs[1] and not has_libs[0]: + return res[1] + raise EasyBuildError("Multiple library subdirectories found for %s in %s: %s", name, root, ', '.join(res)) return res diff --git a/test/framework/modules.py b/test/framework/modules.py index bb11802db7..195bf0339c 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -51,6 +51,7 @@ from easybuild.tools.modules import curr_module_paths, get_software_libdir, get_software_root, get_software_version from easybuild.tools.modules import invalidate_module_caches_for, modules_tool, reset_module_caches from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext # number of modules included for testing purposes @@ -678,11 +679,19 @@ def test_get_software_root_version_libdir(self): root = os.path.join(tmpdir, name) mkdir(os.path.join(root, 'lib64')) os.environ['EBROOT%s' % env_var_name] = root - write_file(os.path.join(root, 'lib', 'foo.a'), 'foo') + write_file(os.path.join(root, 'lib', 'libfoo.a'), 'foo') self.assertEqual(get_software_libdir(name), 'lib') + remove_file(os.path.join(root, 'lib', 'libfoo.a')) + + # also check vice versa with *shared* library in lib64 + shlib_ext = get_shared_lib_ext() + write_file(os.path.join(root, 'lib64', 'libfoo.' + shlib_ext), 'foo') + self.assertEqual(get_software_libdir(name), 'lib64') + + remove_file(os.path.join(root, 'lib64', 'libfoo.' + shlib_ext)) + # check expected result of get_software_libdir with multiple lib subdirs - remove_file(os.path.join(root, 'lib', 'foo.a')) self.assertErrorRegex(EasyBuildError, "Multiple library subdirectories found.*", get_software_libdir, name) self.assertEqual(get_software_libdir(name, only_one=False), ['lib', 'lib64'])