trigger run_shell_cmd hook in run function

easybuild/tools/filetools.py

@@ -58,14 +58,14 @@
 import zlib
 from functools import partial
 from html.parser import HTMLParser
-from importlib.util import spec_from_file_location, module_from_spec
 import urllib.request as std_urllib
 
 from easybuild.base import fancylogger
 # import build_log must stay, to use of EasyBuildLog
 from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning
 from easybuild.tools.config import ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN, build_option, install_path
 from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar
+from easybuild.tools.hooks import load_source
 from easybuild.tools.run import run
 from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg
 
@@ -2771,14 +2771,6 @@ def install_fake_vsc():
     return fake_vsc_path
 
 
-def load_source(filename, path):
-    """Load file as Python module"""
-    spec = spec_from_file_location(filename, path)
-    module = module_from_spec(spec)
-    spec.loader.exec_module(module)
-    return module
-
-
 def get_easyblock_class_name(path):
     """Make sure file is an easyblock and get easyblock class name"""
     fn = os.path.basename(path).split('.')[0]

easybuild/tools/hooks.py

@@ -35,7 +35,7 @@
 from easybuild.base import fancylogger
 from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.config import build_option
-from easybuild.tools.filetools import load_source
+from importlib.util import spec_from_file_location, module_from_spec
 
 
 _log = fancylogger.getLogger('hooks', fname=False)
@@ -118,6 +118,14 @@
 _cached_hooks = {}
 
 
+def load_source(filename, path):
+    """Load file as Python module"""
+    spec = spec_from_file_location(filename, path)
+    module = module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
 def load_hooks(hooks_path):
     """Load defined hooks (if any)."""
 

easybuild/tools/run.py

@@ -110,7 +110,7 @@ def cache_aware_func(cmd, *args, **kwargs):
 @run_cache
 def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None,
         hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, shell=True,
-        output_file=False, stream_output=False, asynchronous=False,
+        output_file=False, stream_output=False, asynchronous=False, with_hooks=True,
         qa_patterns=None, qa_wait_patterns=None):
     """
     Run specified (interactive) shell command, and capture output + exit code.
@@ -127,6 +127,7 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None,
     :param output_file: collect command output in temporary output file
     :param stream_output: stream command output to stdout
     :param asynchronous: run command asynchronously
+    :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined)
     :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers
     :param qa_wait_patterns: list of 2-tuples with patterns for non-questions
                              and number of iterations to allow these patterns to match with end out command output
@@ -135,6 +136,18 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None,
     - exit_code: exit code of command (integer)
     - stderr: stderr output if split_stderr is enabled, None otherwise
     """
+    def to_cmd_str(cmd):
+        """
+        Helper function to create string representation of specified command.
+        """
+        if isinstance(cmd, str):
+            cmd_str = cmd.strip()
+        elif isinstance(cmd, list):
+            cmd_str = ' '.join(cmd)
+        else:
+            raise EasyBuildError(f"Unknown command type ('{type(cmd)}'): {cmd}")
+
+        return cmd_str
 
     # temporarily raise a NotImplementedError until all options are implemented
     if any((stream_output, asynchronous)):
@@ -143,13 +156,6 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None,
     if qa_patterns or qa_wait_patterns:
         raise NotImplementedError
 
-    if isinstance(cmd, str):
-        cmd_str = cmd.strip()
-    elif isinstance(cmd, list):
-        cmd_str = ' '.join(cmd)
-    else:
-        raise EasyBuildError(f"Unknown command type ('{type(cmd)}'): {cmd}")
-
     if work_dir is None:
         work_dir = os.getcwd()
 
@@ -162,6 +168,8 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None,
     else:
         cmd_out_fp = None
 
+    cmd_str = to_cmd_str(cmd)
+
     # early exit in 'dry run' mode, after printing the command that would be run (unless 'hidden' is enabled)
     if not in_dry_run and build_option('extended_dry_run'):
         if not hidden or verbose_dry_run:
@@ -187,6 +195,14 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None,
 
     stderr = subprocess.PIPE if split_stderr else subprocess.STDOUT
 
+    if with_hooks:
+        hooks = load_hooks(build_option('hooks'))
+        hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': work_dir})
+        if hook_res:
+            cmd, old_cmd = hook_res, cmd
+            cmd_str = to_cmd_str(cmd)
+            _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd)
+
     _log.info(f"Running command '{cmd_str}' in {work_dir}")
     proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=stderr, check=fail_on_error,
                           cwd=work_dir, env=env, input=stdin, shell=shell, executable=executable)
@@ -197,6 +213,15 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None,
 
     res = RunResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr_output, work_dir=work_dir)
 
+    if with_hooks:
+        run_hook_kwargs = {
+            'exit_code': res.exit_code,
+            'output': res.output,
+            'stderr': res.stderr,
+            'work_dir': res.work_dir,
+        }
+        run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
+
     if split_stderr:
         log_msg = f"Command '{cmd_str}' exited with exit code {res.exit_code}, "
         log_msg += f"with stdout:\n{res.output}\nstderr:\n{res.stderr}"

easybuild/tools/systemtools.py

@@ -274,7 +274,7 @@ def get_avail_core_count():
         core_cnt = int(sum(sched_getaffinity()))
     else:
         # BSD-type systems
-        res = run('sysctl -n hw.ncpu', in_dry_run=True, hidden=True)
+        res = run('sysctl -n hw.ncpu', in_dry_run=True, hidden=True, with_hooks=False)
         try:
             if int(res.output) > 0:
                 core_cnt = int(res.output)
@@ -311,7 +311,7 @@ def get_total_memory():
     elif os_type == DARWIN:
         cmd = "sysctl -n hw.memsize"
         _log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd)
-        res = run(cmd, in_dry_run=True, hidden=True)
+        res = run(cmd, in_dry_run=True, hidden=True, with_hooks=False)
         if res.exit_code == 0:
             memtotal = int(res.output.strip()) // (1024**2)
 
@@ -393,14 +393,14 @@ def get_cpu_vendor():
 
     elif os_type == DARWIN:
         cmd = "sysctl -n machdep.cpu.vendor"
-        res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True)
+        res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False)
         out = res.output.strip()
         if res.exit_code == 0 and out in VENDOR_IDS:
             vendor = VENDOR_IDS[out]
             _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd))
         else:
             cmd = "sysctl -n machdep.cpu.brand_string"
-            res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True)
+            res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False)
             out = res.output.strip().split(' ')[0]
             if res.exit_code == 0 and out in CPU_VENDORS:
                 vendor = out
@@ -503,7 +503,7 @@ def get_cpu_model():
 
     elif os_type == DARWIN:
         cmd = "sysctl -n machdep.cpu.brand_string"
-        res = run(cmd, in_dry_run=True, hidden=True)
+        res = run(cmd, in_dry_run=True, hidden=True, with_hooks=False)
         if res.exit_code == 0:
             model = res.output.strip()
             _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model))
@@ -548,7 +548,7 @@ def get_cpu_speed():
     elif os_type == DARWIN:
         cmd = "sysctl -n hw.cpufrequency_max"
         _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd)
-        res = run(cmd, in_dry_run=True, hidden=True)
+        res = run(cmd, in_dry_run=True, hidden=True, with_hooks=False)
         out = res.output.strip()
         cpu_freq = None
         if res.exit_code == 0 and out:
@@ -596,7 +596,7 @@ def get_cpu_features():
         for feature_set in ['extfeatures', 'features', 'leaf7_features']:
             cmd = "sysctl -n machdep.cpu.%s" % feature_set
             _log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd)
-            res = run(cmd, in_dry_run=True, hidden=True, fail_on_error=False)
+            res = run(cmd, in_dry_run=True, hidden=True, fail_on_error=False, with_hooks=False)
             if res.exit_code == 0:
                 cpu_feat.extend(res.output.strip().lower().split())
 
@@ -623,7 +623,7 @@ def get_gpu_info():
         try:
             cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader"
             _log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd)
-            res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True)
+            res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False)
             if res.exit_code == 0:
                 for line in res.output.strip().split('\n'):
                     nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {})
@@ -641,13 +641,13 @@ def get_gpu_info():
         try:
             cmd = "rocm-smi --showdriverversion --csv"
             _log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd)
-            res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True)
+            res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False)
             if res.exit_code == 0:
                 amd_driver = res.output.strip().split('\n')[1].split(',')[1]
 
             cmd = "rocm-smi --showproductname --csv"
             _log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd)
-            res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True)
+            res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False)
             if res.exit_code == 0:
                 for line in res.output.strip().split('\n')[1:]:
                     amd_card_series = line.split(',')[1]
@@ -893,7 +893,7 @@ def get_tool_version(tool, version_option='--version', ignore_ec=False):
     Get output of running version option for specific command line tool.
     Output is returned as a single-line string (newlines are replaced by '; ').
     """
-    res = run(' '.join([tool, version_option]), fail_on_error=False, in_dry_run=True, hidden=True)
+    res = run(' '.join([tool, version_option]), fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False)
     if not ignore_ec and res.exit_code:
         _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, res.output))
         return UNKNOWN

test/framework/filetools.py

@@ -49,7 +49,6 @@
 from urllib import request
 from easybuild.tools import run
 import easybuild.tools.filetools as ft
-import easybuild.tools.py2vs3 as py2vs3
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option
 from easybuild.tools.multidiff import multidiff
@@ -3411,17 +3410,6 @@ def test_set_gid_sticky_bits(self):
         self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID)
         self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX)
 
-    def test_compat_makedirs(self):
-        """Test compatibility layer for Python3 os.makedirs"""
-        name = os.path.join(self.test_prefix, 'folder')
-        self.assertNotExists(name)
-        py2vs3.makedirs(name)
-        self.assertExists(name)
-        # exception is raised because file exists (OSError in Python 2, FileExistsError in Python 3)
-        self.assertErrorRegex(Exception, '.*', py2vs3.makedirs, name)
-        py2vs3.makedirs(name, exist_ok=True)  # No error
-        self.assertExists(name)
-
     def test_create_unused_dir(self):
         """Test create_unused_dir function."""
         path = ft.create_unused_dir(self.test_prefix, 'folder')

test/framework/run.py

@@ -1203,6 +1203,9 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
         write_file(hooks_file, hooks_file_txt)
         update_build_option('hooks', hooks_file)
 
+        # disable trace output to make checking of generated output produced by hooks easier
+        update_build_option('trace', False)
+
         with self.mocked_stdout_stderr():
             run_cmd("make")
             stdout = self.get_stdout()
@@ -1225,6 +1228,54 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
         ])
         self.assertEqual(stdout, expected_stdout)
 
+    def test_run_with_hooks(self):
+        """
+        Test running command with run with pre/post run_shell_cmd hooks in place.
+        """
+        cwd = os.getcwd()
+
+        hooks_file = os.path.join(self.test_prefix, 'my_hooks.py')
+        hooks_file_txt = textwrap.dedent("""
+            def pre_run_shell_cmd_hook(cmd, *args, **kwargs):
+                work_dir = kwargs['work_dir']
+                if kwargs.get('interactive'):
+                    print("pre-run hook interactive '||%s||' in %s" % (cmd, work_dir))
+                else:
+                    print("pre-run hook '%s' in %s" % (cmd, work_dir))
+                    import sys
+                    sys.stderr.write('pre-run hook done\\n')
+                if not cmd.startswith('echo'):
+                    cmds = cmd.split(';')
+                    return '; '.join(cmds[:-1] + ["echo " + cmds[-1].lstrip()])
+
+            def post_run_shell_cmd_hook(cmd, *args, **kwargs):
+                exit_code = kwargs.get('exit_code')
+                output = kwargs.get('output')
+                work_dir = kwargs['work_dir']
+                if kwargs.get('interactive'):
+                    msg = "post-run hook interactive '%s'" % cmd
+                else:
+                    msg = "post-run hook '%s'" % cmd
+                msg += " (exit code: %s, output: '%s')" % (exit_code, output)
+                print(msg)
+        """)
+        write_file(hooks_file, hooks_file_txt)
+        update_build_option('hooks', hooks_file)
+
+        # disable trace output to make checking of generated output produced by hooks easier
+        update_build_option('trace', False)
+
+        with self.mocked_stdout_stderr():
+            run("make")
+            stdout = self.get_stdout()
+
+        expected_stdout = '\n'.join([
+            "pre-run hook 'make' in %s" % cwd,
+            "post-run hook 'echo make' (exit code: 0, output: 'make\n')",
+            '',
+        ])
+        self.assertEqual(stdout, expected_stdout)
+
 
 def suite():
     """ returns all the testcases in this module """

test/framework/toy_build.py

@@ -3032,7 +3032,7 @@ def end_hook():
                print('end hook triggered, all done!')
 
             def pre_run_shell_cmd_hook(cmd, *args, **kwargs):
-                if cmd.strip() == TOY_COMP_CMD:
+                if isinstance(cmd, str) and cmd.strip() == TOY_COMP_CMD:
                     print("pre_run_shell_cmd_hook triggered for '%s'" % cmd)
                     # 'copy_toy_file' command doesn't exist, but don't worry,
                     # this problem will be fixed in post_run_shell_cmd_hook
@@ -3043,17 +3043,23 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
                 exit_code = kwargs['exit_code']
                 output = kwargs['output']
                 work_dir = kwargs['work_dir']
-                if cmd.strip().startswith(TOY_COMP_CMD) and exit_code:
+                if isinstance(cmd, str) and cmd.strip().startswith(TOY_COMP_CMD) and exit_code:
                     cwd = change_dir(work_dir)
                     copy_file('toy', 'copy_of_toy')
                     change_dir(cwd)
                     print("'%s' command failed (exit code %s), but I fixed it!" % (cmd, exit_code))
         """)
         write_file(hooks_file, hooks_file_txt)
 
+        extra_args = [
+            '--hooks=%s' % hooks_file,
+            # disable trace output to make checking of generated output produced by hooks easier
+            '--disable-trace',
+        ]
+
         self.mock_stderr(True)
         self.mock_stdout(True)
-        self._test_toy_build(ec_file=test_ec, extra_args=['--hooks=%s' % hooks_file], raise_error=True, debug=False)
+        self._test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True, debug=False)
         stderr = self.get_stderr()
         stdout = self.get_stdout()
         self.mock_stderr(False)
@@ -3074,7 +3080,7 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
         # - for fake module file being created during sanity check (triggered twice, for main + toy install)
         # - for final module file
         # - for devel module file
-        expected_output = textwrap.dedent("""
+        expected_output = textwrap.dedent(f"""
             start hook triggered
             toy 0.0
             ['%(name)s-%(version)s.tar.gz']
@@ -4051,7 +4057,7 @@ def test_toy_build_info_msg(self):
         write_file(test_ec, test_ec_txt)
 
         with self.mocked_stdout_stderr():
-            self.test_toy_build(ec_file=test_ec, testing=False, verify=False, raise_error=True)
+            self._test_toy_build(ec_file=test_ec, testing=False, verify=False, raise_error=True)
             stdout = self.get_stdout()
 
         pattern = '\n'.join([