Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

detect Fortran .mod files in GCCcore installations #4389

Merged
merged 8 commits into from
Jan 3, 2024
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
21 changes: 21 additions & 0 deletions easybuild/framework/easyblock.py
Original file line number Diff line number Diff line change
Expand Up @@ -3303,6 +3303,19 @@ def regex_for_lib(lib):

return fail_msg

def sanity_check_mod_files(self):
"""
Check installation for Fortran .mod files
"""
self.log.debug(f"Checking for .mod files in install directory {self.installdir}...")
mod_files = glob.glob(os.path.join(self.installdir, '**', '*.mod'), recursive=True)

fail_msg = None
if mod_files:
fail_msg = ".mod files (%s) found in the installation." % ', '.join(mod_files)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@branfosj Can we try to filter out false positives here?

I was thinking about checking the output of the file command, if that's available.
Not sure if that makes sense...

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure what we would want to exclude by using file.

$ file 2.3.0-GCCcore-11.2.0/lib64/kim-api/mod/kim_unit_system_module.mod
2.3.0-GCCcore-11.2.0/lib64/kim-api/mod/kim_unit_system_module.mod:                     gzip compressed data, from Unix, original size 57651

$ file Clang/15.0.5-GCCcore-11.3.0/include/flang/omp_lib.mod
Clang/15.0.5-GCCcore-11.3.0/include/flang/omp_lib.mod:                       UTF-8 Unicode (with BOM) text

$ file json-fortran/8.3.0-GCCcore-12.2.0/include/json_value_module.mod
json-fortran/8.3.0-GCCcore-12.2.0/include/json_value_module.mod:     gzip compressed data, from Unix, original size 201261

$ file CPLEX/20.10-GCCcore-10.2.0/opl/examples/opl/cplex_staticLex/staticlex3.mod
CPLEX/20.10-GCCcore-10.2.0/opl/examples/opl/cplex_staticLex/staticlex3.mod: ASCII text

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's always a gzipped file (at least with gfortran), so you could look for that. One could technically unpack it and look for the header

GFORTRAN module version ....

but limiting this to just *.mod and gzipped file should be pretty good.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As discussed, we can refine this later to try and weed out false positives without having to whitelist...


return fail_msg

def _sanity_check_step_common(self, custom_paths, custom_commands):
"""
Determine sanity check paths and commands to use.
Expand Down Expand Up @@ -3625,6 +3638,14 @@ def xs2str(xs):
self.log.warning("Check for required/banned linked shared libraries failed!")
self.sanity_check_fail_msgs.append(linked_shared_lib_fails)

if self.toolchain.name in ['GCCcore'] and not self.cfg['skip_mod_files_sanity_check']:
mod_files_found_msg = self.sanity_check_mod_files()
if mod_files_found_msg:
if build_option('fail_on_mod_files_gcccore'):
self.sanity_check_fail_msgs.append(mod_files_found_msg)
else:
print_warning(mod_files_found_msg)

# cleanup
if self.fake_mod_data:
self.clean_up_fake_module(self.fake_mod_data)
Expand Down
1 change: 1 addition & 0 deletions easybuild/framework/easyconfig/default.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@
'sanity_check_paths': [{}, ("List of files and directories to check "
"(format: {'files':<list>, 'dirs':<list>})"), BUILD],
'skip': [False, "Skip existing software", BUILD],
'skip_mod_files_sanity_check': [False, "Skip the check for .mod files in a GCCcore level install", BUILD],
'skipsteps': [[], "Skip these steps", BUILD],
'source_urls': [[], "List of URLs for source files", BUILD],
'sources': [[], "List of source files", BUILD],
Expand Down
1 change: 1 addition & 0 deletions easybuild/tools/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'enforce_checksums',
'experimental',
'extended_dry_run',
'fail_on_mod_files_gcccore',
'force',
'generate_devel_module',
'group_writable_installdir',
Expand Down
2 changes: 2 additions & 0 deletions easybuild/tools/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,8 @@ def override_options(self):
None, 'store_true', False),
'extra-modules': ("List of extra modules to load after setting up the build environment",
'strlist', 'extend', None),
'fail-on-mod-files-gcccore': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true',
False),
'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, "
"implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False),
'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, "
Expand Down
38 changes: 38 additions & 0 deletions test/framework/toy_build.py
Original file line number Diff line number Diff line change
Expand Up @@ -3955,6 +3955,44 @@ def test_toy_build_sanity_check_linked_libs(self):
self._test_toy_build(ec_file=test_ec, extra_args=args, force=False,
raise_error=True, verbose=False, verify=False)

def test_toy_mod_files(self):
"""Check detection of .mod files"""
test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs')
toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
test_ec_txt = read_file(toy_ec)
test_ec = os.path.join(self.test_prefix, 'test.eb')
write_file(test_ec, test_ec_txt)

with self.mocked_stdout_stderr():
self._test_toy_build(ec_file=test_ec)

test_ec_txt += "\npostinstallcmds += ['touch %(installdir)s/lib/file.mod']"
write_file(test_ec, test_ec_txt)

with self.mocked_stdout_stderr():
self._test_toy_build(ec_file=test_ec)

args = ['--try-toolchain=GCCcore,6.2.0', '--disable-map-toolchains']
self.mock_stdout(True)
self.mock_stderr(True)
self._test_toy_build(ec_file=test_ec, extra_args=args)
stderr = self.get_stderr()
self.mock_stdout(False)
self.mock_stderr(False)
pattern = r"WARNING: .mod files (.*) found in the installation."
self.assertRegex(stderr.strip(), pattern)

args += ['--fail-on-mod-files-gcccore']
pattern = r"Sanity check failed: .mod files (.*) found in the installation."
self.assertErrorRegex(EasyBuildError, pattern, self.run_test_toy_build_with_output, ec_file=test_ec,
extra_args=args, verify=False, fails=True, verbose=False, raise_error=True)

test_ec_txt += "\nskip_mod_files_sanity_check = True"
write_file(test_ec, test_ec_txt)

with self.mocked_stdout_stderr():
self._test_toy_build(ec_file=test_ec, extra_args=args)

def test_toy_ignore_test_failure(self):
"""Check whether use of --ignore-test-failure is mentioned in build output."""
args = ['--ignore-test-failure']
Expand Down