diff --git a/emmake.py b/emmake.py index 56d6b499a5f47..99c58bf454b86 100755 --- a/emmake.py +++ b/emmake.py @@ -26,7 +26,7 @@ import shutil import sys -from tools import building, shared, utils +from tools import building, utils # @@ -58,10 +58,10 @@ def run(): # 'sdl2-config.bat' in PATH. print(f'emmake: "{shlex.join(args)}" in "{os.getcwd()}"', file=sys.stderr) if utils.WINDOWS: - return shared.run_process(args, check=False, shell=True, env=env).returncode + return utils.run_process(args, check=False, shell=True, env=env).returncode else: os.environ.update(env) - shared.exec_process(args) + utils.exec(args) if __name__ == '__main__': diff --git a/test/benchmark/benchmark_sse.py b/test/benchmark/benchmark_sse.py index 5bc6ce281c541..2e6e0bb8e0295 100644 --- a/test/benchmark/benchmark_sse.py +++ b/test/benchmark/benchmark_sse.py @@ -21,7 +21,8 @@ from common import EMRUN, test_file from tools.config import V8_ENGINE -from tools.shared import CLANG_CXX, EMCC, WINDOWS, run_process +from tools.shared import CLANG_CXX, EMCC, WINDOWS +from tools.utils import run_process # System info system_info = subprocess.check_output([EMRUN, '--system_info'], stderr=subprocess.STDOUT, text=True) diff --git a/test/clang_native.py b/test/clang_native.py index a58a911f9be65..e4dcc61025db8 100644 --- a/test/clang_native.py +++ b/test/clang_native.py @@ -8,8 +8,8 @@ import platform import sys -from tools.shared import CLANG_CC, CLANG_CXX, PIPE, run_process -from tools.utils import MACOS, WINDOWS, path_from_root +from tools.shared import CLANG_CC, CLANG_CXX, PIPE +from tools.utils import MACOS, WINDOWS, path_from_root, run_process logger = logging.getLogger('clang_native') diff --git a/test/common.py b/test/common.py index 53db5dc7971a7..cd4ca7db6deea 100644 --- a/test/common.py +++ b/test/common.py @@ -959,7 +959,7 @@ def verify_es5(self, filename): # ES-Check: there were no ES version matching errors! # pipe stdout and stderr so that we can choose if/when to print this # output and avoid spamming stdout when tests are successful. - shared.run_process(es_check + ['es5', inputfile], stdout=PIPE, stderr=STDOUT, env=es_check_env) + utils.run_process(es_check + ['es5', inputfile], stdout=PIPE, stderr=STDOUT, env=es_check_env) except subprocess.CalledProcessError as e: print(e.stdout) self.fail('es-check failed to verify ES5 output compliance') @@ -1296,7 +1296,7 @@ def clear(self): utils.delete_contents(shared.EMSCRIPTEN_TEMP_DIR) def run_process(self, cmd, check=True, **kwargs): - # Wrapper around shared.run_process. This is desirable so that the tests + # Wrapper around utils.run_process. This is desirable so that the tests # can fail (in the unittest sense) rather than error'ing. # In the long run it would nice to completely remove the dependency on # core emscripten code (shared.py) here. @@ -1312,7 +1312,7 @@ def run_process(self, cmd, check=True, **kwargs): kwargs['stderr'] = PIPE try: - rtn = shared.run_process(cmd, check=check, **kwargs) + rtn = utils.run_process(cmd, check=check, **kwargs) except subprocess.CalledProcessError as e: if check and e.returncode != 0: print(e.stdout) @@ -2349,8 +2349,7 @@ def build_library(name, with open(os.path.join(project_dir, 'configure_err'), 'w') as err: stdout = out if EMTEST_BUILD_VERBOSE < 2 else None stderr = err if EMTEST_BUILD_VERBOSE < 1 else None - shared.run_process(configure, env=env, stdout=stdout, stderr=stderr, - cwd=project_dir) + utils.run_process(configure, env=env, stdout=stdout, stderr=stderr, cwd=project_dir) except subprocess.CalledProcessError: print('-- configure stdout --') print(read_file(Path(project_dir, 'configure_out'))) @@ -2378,8 +2377,7 @@ def open_make_err(mode='r'): with open_make_err('w') as make_err: stdout = make_out if EMTEST_BUILD_VERBOSE < 2 else None stderr = make_err if EMTEST_BUILD_VERBOSE < 1 else None - shared.run_process(make + make_args, stdout=stdout, stderr=stderr, env=env, - cwd=project_dir) + utils.run_process(make + make_args, stdout=stdout, stderr=stderr, env=env, cwd=project_dir) except subprocess.CalledProcessError: with open_make_out() as f: print('-- make stdout --') diff --git a/test/test_benchmark.py b/test/test_benchmark.py index 993fd4f000d5a..c0d98a9481b60 100644 --- a/test/test_benchmark.py +++ b/test/test_benchmark.py @@ -25,7 +25,8 @@ from decorators import needs_make from tools import building, utils -from tools.shared import CLANG_CC, CLANG_CXX, EMCC, PIPE, config, run_process +from tools.shared import CLANG_CC, CLANG_CXX, EMCC, PIPE, config +from tools.utils import run_process # standard arguments for timing: # 0: no runtime, just startup diff --git a/test/test_other.py b/test/test_other.py index b5ba216fa12af..380d35af0814e 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -268,7 +268,7 @@ def decorated(self, *args, **kwargs): def llvm_nm(file): - output = shared.run_process([LLVM_NM, file], stdout=PIPE).stdout + output = utils.run_process([LLVM_NM, file], stdout=PIPE).stdout symbols = { 'defs': set(), diff --git a/test/test_sockets.py b/test/test_sockets.py index 61381d38e31e6..d152b634b0575 100644 --- a/test/test_sockets.py +++ b/test/test_sockets.py @@ -28,7 +28,8 @@ ) from tools import config -from tools.shared import CLANG_CC, EMCC, path_from_root, run_process +from tools.shared import CLANG_CC, EMCC, path_from_root +from tools.utils import run_process npm_checked = False diff --git a/tools/building.py b/tools/building.py index e2afec294d157..b119f06acdd4c 100644 --- a/tools/building.py +++ b/tools/building.py @@ -46,10 +46,9 @@ get_emscripten_temp_dir, is_c_symbol, path_from_root, - run_process, ) from .toolchain_profiler import ToolchainProfiler -from .utils import WINDOWS +from .utils import WINDOWS, run_process logger = logging.getLogger('building') diff --git a/tools/cmdline.py b/tools/cmdline.py index 50a5f561bce09..4f7a195f0abaa 100644 --- a/tools/cmdline.py +++ b/tools/cmdline.py @@ -145,7 +145,7 @@ def version_string(): # look up and find the revision in a parent directory that is a git repo revision_suffix = '' if os.path.exists(utils.path_from_root('.git')): - git_rev = shared.run_process( + git_rev = utils.run_process( ['git', 'rev-parse', 'HEAD'], stdout=PIPE, stderr=PIPE, cwd=utils.path_from_root()).stdout.strip() revision_suffix = ' (%s)' % git_rev diff --git a/tools/empath-split.py b/tools/empath-split.py index d92e1c98c536d..1a9fc0e5029f9 100755 --- a/tools/empath-split.py +++ b/tools/empath-split.py @@ -68,7 +68,7 @@ __rootdir__ = os.path.dirname(__scriptdir__) sys.path.insert(0, __rootdir__) -from tools import building, diagnostics, emsymbolizer, shared, utils, webassembly +from tools import building, diagnostics, emsymbolizer, utils, webassembly from tools.utils import exit_with_error @@ -349,7 +349,7 @@ def main(): cmd += forwarded_args if args.verbose: print('\n' + ' '.join(cmd)) - shared.run_process(cmd) + utils.run_process(cmd) finally: if not args.preserve_manifest: os.remove(manifest) diff --git a/tools/emsymbolizer.py b/tools/emsymbolizer.py index 0adcbfc69c460..c71fc26da890d 100755 --- a/tools/emsymbolizer.py +++ b/tools/emsymbolizer.py @@ -26,7 +26,7 @@ __rootdir__ = os.path.dirname(__scriptdir__) sys.path.insert(0, __rootdir__) -from tools import shared, webassembly +from tools import shared, utils, webassembly LLVM_SYMBOLIZER = shared.llvm_tool_path('llvm-symbolizer') @@ -77,7 +77,7 @@ def symbolize_address_symbolizer(module, address, is_dwarf): str(address)] if shared.DEBUG: print(f'Running {" ".join(cmd)}') - out = shared.run_process(cmd, stdout=subprocess.PIPE).stdout.strip() + out = utils.run_process(cmd, stdout=subprocess.PIPE).stdout.strip() out_lines = out.splitlines() # Source location regex, e.g., /abc/def.c:3:5 diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 6392fa98b06da..cb3836a074651 100755 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -99,7 +99,7 @@ def minify_shell(self, shell, minify_whitespace): cmd = get_acorn_cmd() + [temp_file, 'minifyGlobals'] if minify_whitespace: cmd.append('--minify-whitespace') - output = shared.run_process(cmd, stdout=subprocess.PIPE).stdout + output = utils.run_process(cmd, stdout=subprocess.PIPE).stdout assert len(output) and not output.startswith('Assertion failed'), 'Error in js optimizer: ' + output code, metadata = output.split('// EXTRA_INFO:') diff --git a/tools/shared.py b/tools/shared.py index fe428a10c44dd..031b0f17a82f5 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -100,26 +100,6 @@ diagnostics.add_warning('closure', enabled=False) -def run_process(cmd, check=True, input=None, *args, **kw): - """Runs a subprocess returning the exit code. - - By default this function will raise an exception on failure. Therefore this should only be - used if you want to handle such failures. For most subprocesses, failures are not recoverable - and should be fatal. In those cases the `check_call` wrapper should be preferred. - """ - - # Flush standard streams otherwise the output of the subprocess may appear in the - # output before messages that we have already written. - sys.stdout.flush() - sys.stderr.flush() - kw.setdefault('text', True) - kw.setdefault('encoding', 'utf-8') - ret = subprocess.run(cmd, check=check, input=input, *args, **kw) - debug_text = '%sexecuted %s' % ('successfully ' if check else '', shlex.join(cmd)) - logger.debug(debug_text) - return ret - - def returncode_to_str(code): assert code != 0 if code < 0: @@ -210,7 +190,7 @@ def check_call(cmd, *args, **kw): if SKIP_SUBPROCS: return 0 try: - return run_process(cmd, *args, **kw) + return utils.run_process(cmd, *args, **kw) except subprocess.CalledProcessError as e: exit_with_error("'%s' failed (%s)", shlex.join(cmd), returncode_to_str(e.returncode)) except OSError as e: @@ -219,13 +199,7 @@ def check_call(cmd, *args, **kw): def exec_process(cmd): print_compiler_stage(cmd) - if utils.WINDOWS: - rtn = run_process(cmd, stdin=sys.stdin, check=False).returncode - sys.exit(rtn) - else: - sys.stdout.flush() - sys.stderr.flush() - os.execvp(cmd[0], cmd) + utils.exec(cmd) def run_js_tool(filename, jsargs=[], node_args=[], **kw): # noqa: B006 @@ -278,7 +252,7 @@ def get_clang_targets(): if not os.path.exists(CLANG_CC): exit_with_error('clang executable not found at `%s`' % CLANG_CC) try: - target_info = run_process([CLANG_CC, '-print-targets'], stdout=PIPE).stdout + target_info = utils.run_process([CLANG_CC, '-print-targets'], stdout=PIPE).stdout except subprocess.CalledProcessError: exit_with_error('error running `clang -print-targets`. Check your llvm installation (%s)' % CLANG_CC) if 'Registered Targets:' not in target_info: @@ -312,7 +286,7 @@ def env_with_node_in_path(): def _get_node_version_pair(nodejs): - actual = run_process(nodejs + ['--version'], stdout=PIPE).stdout.strip() + actual = utils.run_process(nodejs + ['--version'], stdout=PIPE).stdout.strip() version = actual.replace('v', '') version = version.split('-')[0].split('.') version = tuple(int(v) for v in version) @@ -371,7 +345,7 @@ def node_pthread_flags(nodejs): @ToolchainProfiler.profile() def check_node(): try: - run_process(config.NODE_JS + ['-e', 'console.log("hello")'], stdout=PIPE) + utils.run_process(config.NODE_JS + ['-e', 'console.log("hello")'], stdout=PIPE) except Exception as e: exit_with_error('the configured node executable (%s) does not seem to work, check the paths in %s (%s)', config.NODE_JS, config.EM_CONFIG, str(e)) diff --git a/tools/utils.py b/tools/utils.py index 14e0c1ec1b8f8..7550f0a6e245b 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -10,8 +10,10 @@ import functools import logging import os +import shlex import shutil import stat +import subprocess import sys from pathlib import Path @@ -25,6 +27,36 @@ logger = logging.getLogger('utils') +def run_process(cmd, check=True, input=None, *args, **kw): + """Runs a subprocess returning the exit code. + + By default this function will raise an exception on failure. Therefore this should only be + used if you want to handle such failures. For most subprocesses, failures are not recoverable + and should be fatal. In those cases the `check_call` wrapper should be preferred. + """ + + # Flush standard streams otherwise the output of the subprocess may appear in the + # output before messages that we have already written. + sys.stdout.flush() + sys.stderr.flush() + kw.setdefault('text', True) + kw.setdefault('encoding', 'utf-8') + ret = subprocess.run(cmd, check=check, input=input, *args, **kw) + debug_text = '%sexecuted %s' % ('successfully ' if check else '', shlex.join(cmd)) + logger.debug(debug_text) + return ret + + +def exec(cmd): + if WINDOWS: + rtn = run_process(cmd, stdin=sys.stdin, check=False).returncode + sys.exit(rtn) + else: + sys.stdout.flush() + sys.stderr.flush() + os.execvp(cmd[0], cmd) + + def exit_with_error(msg, *args): diagnostics.error(msg, *args)