diff --git a/test/common.py b/test/common.py index 76d7a0f3abeef..a73287f8589d3 100644 --- a/test/common.py +++ b/test/common.py @@ -7,7 +7,6 @@ import difflib import hashlib import io -import itertools import json import logging import os @@ -23,7 +22,6 @@ import textwrap import threading import time -import unittest import webbrowser from enum import Enum from functools import wraps @@ -43,7 +41,6 @@ from tools.settings import COMPILE_TIME_SETTINGS from tools.shared import DEBUG, EMCC, EMXX, get_canonical_temp_dir, path_from_root from tools.utils import ( - MACOS, WINDOWS, exit_with_error, read_binary, @@ -193,8 +190,6 @@ def configure(data_dir): if not config.JS_ENGINES: config.JS_ENGINES = [config.NODE_JS_TEST] -requires_network = unittest.skipIf(os.getenv('EMTEST_SKIP_NETWORK_TESTS'), 'This test requires network access') - def errlog(*args): """Shorthand for print with file=sys.stderr @@ -263,127 +258,11 @@ def compiler_for(filename, force_c=False): return EMCC -# Generic decorator that calls a function named 'condition' on the test class and -# skips the test if that function returns true -def skip_if_simple(name, condition, note=''): - assert callable(condition) - assert not callable(note) - - def decorator(func): - assert callable(func) - - @wraps(func) - def decorated(self, *args, **kwargs): - if condition(self): - explanation_str = name - if note: - explanation_str += ': %s' % note - self.skipTest(explanation_str) - return func(self, *args, **kwargs) - - return decorated - - return decorator - - -# Same as skip_if_simple but creates a decorator that takes a note as an argument. -def skip_if(name, condition, default_note=''): - assert callable(condition) - - def decorator(note=default_note): - return skip_if_simple(name, condition, note) - - return decorator - - -def is_slow_test(func): - assert callable(func) - decorated = skip_if_simple('skipping slow tests', lambda _: EMTEST_SKIP_SLOW)(func) - decorated.is_slow = True - return decorated - - def record_flaky_test(test_name, attempt_count, max_attempts, exception_msg): - logging.info(f'Retrying flaky test "{test_name}" (attempt {attempt_count}/{max_attempts} failed):\n{exception_msg}') + logger.info(f'Retrying flaky test "{test_name}" (attempt {attempt_count}/{max_attempts} failed):\n{exception_msg}') open(flaky_tests_log_filename, 'a').write(f'{test_name}\n') -def flaky(note=''): - assert not callable(note) - - if EMTEST_SKIP_FLAKY: - return unittest.skip(note) - - if not EMTEST_RETRY_FLAKY: - return lambda f: f - - def decorated(func): - @wraps(func) - def modified(self, *args, **kwargs): - # Browser tests have there own method of retrying tests. - if self.is_browser_test(): - self.flaky = True - return func(self, *args, **kwargs) - - for i in range(EMTEST_RETRY_FLAKY): - try: - return func(self, *args, **kwargs) - except (AssertionError, subprocess.TimeoutExpired) as exc: - preserved_exc = exc - record_flaky_test(self.id(), i, EMTEST_RETRY_FLAKY, exc) - - raise AssertionError('Flaky test has failed too many times') from preserved_exc - - return modified - - return decorated - - -def disabled(note=''): - assert not callable(note) - return unittest.skip(note) - - -no_mac = skip_if('no_mac', lambda _: MACOS) - -no_windows = skip_if('no_windows', lambda _: WINDOWS) - -no_wasm64 = skip_if('no_wasm64', lambda t: t.is_wasm64()) - -# 2200mb is the value used by the core_2gb test mode -no_2gb = skip_if('no_2gb', lambda t: t.get_setting('INITIAL_MEMORY') == '2200mb') - -no_4gb = skip_if('no_4gb', lambda t: t.is_4gb()) - -only_windows = skip_if('only_windows', lambda _: not WINDOWS) - -requires_native_clang = skip_if_simple('native clang tests are disabled', lambda _: EMTEST_LACKS_NATIVE_CLANG) - -needs_make = skip_if('tool not available on windows bots', lambda _: WINDOWS) - - -def requires_node(func): - assert callable(func) - - @wraps(func) - def decorated(self, *args, **kwargs): - self.require_node() - return func(self, *args, **kwargs) - - return decorated - - -def requires_node_canary(func): - assert callable(func) - - @wraps(func) - def decorated(self, *args, **kwargs): - self.require_node_canary() - return func(self, *args, **kwargs) - - return decorated - - def node_bigint_flags(node_version): # The --experimental-wasm-bigint flag was added in v12, and then removed (enabled by default) # in v16. @@ -393,100 +272,6 @@ def node_bigint_flags(node_version): return [] -# Used to mark dependencies in various tests to npm developer dependency -# packages, which might not be installed on Emscripten end users' systems. -def requires_dev_dependency(package): - assert not callable(package) - note = f'requires npm development package "{package}" and EMTEST_SKIP_NODE_DEV_PACKAGES is set' - return skip_if_simple('requires_dev_dependency', lambda _: 'EMTEST_SKIP_NODE_DEV_PACKAGES' in os.environ, note) - - -def requires_wasm64(func): - assert callable(func) - - @wraps(func) - def decorated(self, *args, **kwargs): - self.require_wasm64() - return func(self, *args, **kwargs) - - return decorated - - -def requires_wasm_eh(func): - assert callable(func) - - @wraps(func) - def decorated(self, *args, **kwargs): - self.require_wasm_eh() - return func(self, *args, **kwargs) - - return decorated - - -def requires_v8(func): - assert callable(func) - - @wraps(func) - def decorated(self, *args, **kwargs): - self.require_v8() - return func(self, *args, **kwargs) - - return decorated - - -def requires_wasm2js(func): - assert callable(func) - - @wraps(func) - def decorated(self, *args, **kwargs): - self.require_wasm2js() - return func(self, *args, **kwargs) - - return decorated - - -def requires_jspi(func): - assert callable(func) - - @wraps(func) - def decorated(self, *args, **kwargs): - self.require_jspi() - return func(self, *args, **kwargs) - - return decorated - - -def node_pthreads(func): - assert callable(func) - - @wraps(func) - def decorated(self, *args, **kwargs): - self.setup_node_pthreads() - return func(self, *args, **kwargs) - return decorated - - -def crossplatform(func): - assert callable(func) - func.is_crossplatform_test = True - return func - - -# without EMTEST_ALL_ENGINES set we only run tests in a single VM by -# default. in some tests we know that cross-VM differences may happen and -# so are worth testing, and they should be marked with this decorator -def all_engines(func): - assert callable(func) - - @wraps(func) - def decorated(self, *args, **kwargs): - self.use_all_engines = True - self.set_setting('ENVIRONMENT', 'web,node,shell') - return func(self, *args, **kwargs) - - return decorated - - @contextlib.contextmanager def env_modify(updates): """A context manager that updates os.environ.""" @@ -508,339 +293,6 @@ def env_modify(updates): os.environ.update(old_env) -# Decorator version of env_modify -def with_env_modify(updates): - assert not callable(updates) - - def decorated(func): - @wraps(func) - def modified(self, *args, **kwargs): - with env_modify(updates): - return func(self, *args, **kwargs) - return modified - - return decorated - - -def also_with_wasmfs(func): - assert callable(func) - - @wraps(func) - def metafunc(self, wasmfs, *args, **kwargs): - if DEBUG: - print('parameterize:wasmfs=%d' % wasmfs) - if wasmfs: - self.setup_wasmfs_test() - else: - self.cflags += ['-DMEMFS'] - return func(self, *args, **kwargs) - - parameterize(metafunc, {'': (False,), - 'wasmfs': (True,)}) - return metafunc - - -def also_with_nodefs(func): - @wraps(func) - def metafunc(self, fs, *args, **kwargs): - if DEBUG: - print('parameterize:fs=%s' % (fs)) - if fs == 'nodefs': - self.setup_nodefs_test() - else: - self.cflags += ['-DMEMFS'] - assert fs is None - return func(self, *args, **kwargs) - - parameterize(metafunc, {'': (None,), - 'nodefs': ('nodefs',)}) - return metafunc - - -def also_with_nodefs_both(func): - @wraps(func) - def metafunc(self, fs, *args, **kwargs): - if DEBUG: - print('parameterize:fs=%s' % (fs)) - if fs == 'nodefs': - self.setup_nodefs_test() - elif fs == 'rawfs': - self.setup_noderawfs_test() - else: - self.cflags += ['-DMEMFS'] - assert fs is None - return func(self, *args, **kwargs) - - parameterize(metafunc, {'': (None,), - 'nodefs': ('nodefs',), - 'rawfs': ('rawfs',)}) - return metafunc - - -def with_all_fs(func): - @wraps(func) - def metafunc(self, wasmfs, fs, *args, **kwargs): - if DEBUG: - print('parameterize:fs=%s' % (fs)) - if wasmfs: - self.setup_wasmfs_test() - if fs == 'nodefs': - self.setup_nodefs_test() - elif fs == 'rawfs': - self.setup_noderawfs_test() - else: - self.cflags += ['-DMEMFS'] - assert fs is None - return func(self, *args, **kwargs) - - parameterize(metafunc, {'': (False, None), - 'nodefs': (False, 'nodefs'), - 'rawfs': (False, 'rawfs'), - 'wasmfs': (True, None), - 'wasmfs_nodefs': (True, 'nodefs'), - 'wasmfs_rawfs': (True, 'rawfs')}) - return metafunc - - -def also_with_noderawfs(func): - assert callable(func) - - @wraps(func) - def metafunc(self, rawfs, *args, **kwargs): - if DEBUG: - print('parameterize:rawfs=%d' % rawfs) - if rawfs: - self.setup_noderawfs_test() - else: - self.cflags += ['-DMEMFS'] - return func(self, *args, **kwargs) - - parameterize(metafunc, {'': (False,), - 'rawfs': (True,)}) - return metafunc - - -def also_with_minimal_runtime(func): - assert callable(func) - - @wraps(func) - def metafunc(self, with_minimal_runtime, *args, **kwargs): - if DEBUG: - print('parameterize:minimal_runtime=%s' % with_minimal_runtime) - if self.get_setting('MINIMAL_RUNTIME'): - self.skipTest('MINIMAL_RUNTIME already enabled in test config') - if with_minimal_runtime: - if self.get_setting('MODULARIZE') == 'instance' or self.get_setting('WASM_ESM_INTEGRATION'): - self.skipTest('MODULARIZE=instance is not compatible with MINIMAL_RUNTIME') - self.set_setting('MINIMAL_RUNTIME', 1) - # This extra helper code is needed to cleanly handle calls to exit() which throw - # an ExitCode exception. - self.cflags += ['--pre-js', test_file('minimal_runtime_exit_handling.js')] - return func(self, *args, **kwargs) - - parameterize(metafunc, {'': (False,), - 'minimal_runtime': (True,)}) - return metafunc - - -def also_without_bigint(func): - assert callable(func) - - @wraps(func) - def metafunc(self, no_bigint, *args, **kwargs): - if DEBUG: - print('parameterize:no_bigint=%s' % no_bigint) - if no_bigint: - if self.get_setting('WASM_BIGINT') is not None: - self.skipTest('redundant in bigint test config') - self.set_setting('WASM_BIGINT', 0) - return func(self, *args, **kwargs) - - parameterize(metafunc, {'': (False,), - 'no_bigint': (True,)}) - return metafunc - - -def also_with_wasm64(func): - assert callable(func) - - @wraps(func) - def metafunc(self, with_wasm64, *args, **kwargs): - if DEBUG: - print('parameterize:wasm64=%s' % with_wasm64) - if with_wasm64: - self.require_wasm64() - self.set_setting('MEMORY64') - return func(self, *args, **kwargs) - - parameterize(metafunc, {'': (False,), - 'wasm64': (True,)}) - return metafunc - - -def also_with_wasm2js(func): - assert callable(func) - - @wraps(func) - def metafunc(self, with_wasm2js, *args, **kwargs): - assert self.get_setting('WASM') is None - if DEBUG: - print('parameterize:wasm2js=%s' % with_wasm2js) - if with_wasm2js: - self.require_wasm2js() - self.set_setting('WASM', 0) - return func(self, *args, **kwargs) - - parameterize(metafunc, {'': (False,), - 'wasm2js': (True,)}) - return metafunc - - -def can_do_standalone(self, impure=False): - # Pure standalone engines don't support MEMORY64 yet. Even with MEMORY64=2 (lowered) - # the WASI APIs that take pointer values don't have 64-bit variants yet. - if not impure: - if self.get_setting('MEMORY64'): - return False - # This is way to detect the core_2gb test mode in test_core.py - if self.get_setting('INITIAL_MEMORY') == '2200mb': - return False - return self.is_wasm() and \ - self.get_setting('STACK_OVERFLOW_CHECK', 0) < 2 and \ - not self.get_setting('MINIMAL_RUNTIME') and \ - not self.get_setting('WASM_ESM_INTEGRATION') and \ - not self.get_setting('SAFE_HEAP') and \ - not any(a.startswith('-fsanitize=') for a in self.cflags) - - -# Impure means a test that cannot run in a wasm VM yet, as it is not 100% -# standalone. We can still run them with the JS code though. -def also_with_standalone_wasm(impure=False): - def decorated(func): - @wraps(func) - def metafunc(self, standalone, *args, **kwargs): - if DEBUG: - print('parameterize:standalone=%s' % standalone) - if standalone: - if not can_do_standalone(self, impure): - self.skipTest('Test configuration is not compatible with STANDALONE_WASM') - self.set_setting('STANDALONE_WASM') - if not impure: - self.set_setting('PURE_WASI') - self.cflags.append('-Wno-unused-command-line-argument') - # if we are impure, disallow all wasm engines - if impure: - self.wasm_engines = [] - return func(self, *args, **kwargs) - - parameterize(metafunc, {'': (False,), - 'standalone': (True,)}) - return metafunc - - return decorated - - -def also_with_asan(func): - assert callable(func) - - @wraps(func) - def metafunc(self, asan, *args, **kwargs): - if asan: - if self.is_wasm64(): - self.skipTest('TODO: ASAN in memory64') - if self.is_2gb() or self.is_4gb(): - self.skipTest('asan doesnt support GLOBAL_BASE') - self.cflags.append('-fsanitize=address') - return func(self, *args, **kwargs) - - parameterize(metafunc, {'': (False,), - 'asan': (True,)}) - return metafunc - - -def also_with_modularize(func): - assert callable(func) - - @wraps(func) - def metafunc(self, modularize, *args, **kwargs): - if modularize: - if self.get_setting('DECLARE_ASM_MODULE_EXPORTS') == 0: - self.skipTest('DECLARE_ASM_MODULE_EXPORTS=0 is not compatible with MODULARIZE') - if self.get_setting('STRICT_JS'): - self.skipTest('MODULARIZE is not compatible with STRICT_JS') - if self.get_setting('WASM_ESM_INTEGRATION'): - self.skipTest('MODULARIZE is not compatible with WASM_ESM_INTEGRATION') - self.cflags += ['--extern-post-js', test_file('modularize_post_js.js'), '-sMODULARIZE'] - return func(self, *args, **kwargs) - - parameterize(metafunc, {'': (False,), - 'modularize': (True,)}) - return metafunc - - -# Tests exception handling and setjmp/longjmp handling. This tests three -# combinations: -# - Emscripten EH + Emscripten SjLj -# - Wasm EH + Wasm SjLj -# - Wasm EH + Wasm SjLj (Legacy) -def with_all_eh_sjlj(func): - assert callable(func) - - @wraps(func) - def metafunc(self, mode, *args, **kwargs): - if DEBUG: - print('parameterize:eh_mode=%s' % mode) - if mode in {'wasm', 'wasm_legacy'}: - # Wasm EH is currently supported only in wasm backend and V8 - if self.is_wasm2js(): - self.skipTest('wasm2js does not support wasm EH/SjLj') - self.cflags.append('-fwasm-exceptions') - self.set_setting('SUPPORT_LONGJMP', 'wasm') - if mode == 'wasm': - self.require_wasm_eh() - if mode == 'wasm_legacy': - self.require_wasm_legacy_eh() - else: - self.set_setting('DISABLE_EXCEPTION_CATCHING', 0) - self.set_setting('SUPPORT_LONGJMP', 'emscripten') - # DISABLE_EXCEPTION_CATCHING=0 exports __cxa_can_catch, - # so if we don't build in C++ mode, wasm-ld will - # error out because libc++abi is not included. See - # https://github.com/emscripten-core/emscripten/pull/14192 for details. - self.set_setting('DEFAULT_TO_CXX') - return func(self, *args, **kwargs) - - parameterize(metafunc, {'emscripten': ('emscripten',), - 'wasm': ('wasm',), - 'wasm_legacy': ('wasm_legacy',)}) - return metafunc - - -# This works just like `with_all_eh_sjlj` above but doesn't enable exceptions. -# Use this for tests that use setjmp/longjmp but not exceptions handling. -def with_all_sjlj(func): - assert callable(func) - - @wraps(func) - def metafunc(self, mode, *args, **kwargs): - if mode in {'wasm', 'wasm_legacy'}: - if self.is_wasm2js(): - self.skipTest('wasm2js does not support wasm SjLj') - self.set_setting('SUPPORT_LONGJMP', 'wasm') - if mode == 'wasm': - self.require_wasm_eh() - if mode == 'wasm_legacy': - self.require_wasm_legacy_eh() - else: - self.set_setting('SUPPORT_LONGJMP', 'emscripten') - return func(self, *args, **kwargs) - - parameterize(metafunc, {'emscripten': ('emscripten',), - 'wasm': ('wasm',), - 'wasm_legacy': ('wasm_legacy',)}) - return metafunc - - def ensure_dir(dirname): dirname = Path(dirname) dirname.mkdir(parents=True, exist_ok=True) @@ -946,50 +398,6 @@ def find_browser_test_file(filename): return filename -def parameterize(func, parameters): - """Add additional parameterization to a test function. - - This function create or adds to the `_parameterize` property of a function - which is then expanded by the RunnerMeta metaclass into multiple separate - test functions. - """ - prev = getattr(func, '_parameterize', None) - assert not any(p.startswith('_') for p in parameters) - if prev: - # If we're parameterizing 2nd time, construct a cartesian product for various combinations. - func._parameterize = { - '_'.join(filter(None, [k1, k2])): v2 + v1 for (k1, v1), (k2, v2) in itertools.product(prev.items(), parameters.items()) - } - else: - func._parameterize = parameters - - -def parameterized(parameters): - """ - Mark a test as parameterized. - - Usage: - @parameterized({ - 'subtest1': (1, 2, 3), - 'subtest2': (4, 5, 6), - }) - def test_something(self, a, b, c): - ... # actual test body - - This is equivalent to defining two tests: - - def test_something_subtest1(self): - # runs test_something(1, 2, 3) - - def test_something_subtest2(self): - # runs test_something(4, 5, 6) - """ - def decorator(func): - parameterize(func, parameters) - return func - return decorator - - def get_output_suffix(args): if any(a in args for a in ('-sEXPORT_ES6', '-sWASM_ESM_INTEGRATION', '-sMODULARIZE=instance')): return '.mjs' diff --git a/test/decorators.py b/test/decorators.py new file mode 100644 index 0000000000000..51548221dd0eb --- /dev/null +++ b/test/decorators.py @@ -0,0 +1,611 @@ +# Copyright 2025 The Emscripten Authors. All rights reserved. +# Emscripten is available under two separate licenses, the MIT license and the +# University of Illinois/NCSA Open Source License. Both these licenses can be +# found in the LICENSE file. + +"""Common decorators shared between the various suites. + +When decorators are only used a single file they can be defined locally. If they +are used from multiple locations they should be defined here. +""" + +import itertools +import os +import subprocess +import unittest +from functools import wraps + +import common +from common import test_file + +from tools.shared import DEBUG +from tools.utils import MACOS, WINDOWS + +requires_network = unittest.skipIf(os.getenv('EMTEST_SKIP_NETWORK_TESTS'), 'This test requires network access') + + +# Generic decorator that calls a function named 'condition' on the test class and +# skips the test if that function returns true +def skip_if_simple(name, condition, note=''): + assert callable(condition) + assert not callable(note) + + def decorator(func): + assert callable(func) + + @wraps(func) + def decorated(self, *args, **kwargs): + if condition(self): + explanation_str = name + if note: + explanation_str += ': %s' % note + self.skipTest(explanation_str) + return func(self, *args, **kwargs) + + return decorated + + return decorator + + +# Same as skip_if_simple but creates a decorator that takes a note as an argument. +def skip_if(name, condition, default_note=''): + assert callable(condition) + + def decorator(note=default_note): + return skip_if_simple(name, condition, note) + + return decorator + + +def is_slow_test(func): + assert callable(func) + decorated = skip_if_simple('skipping slow tests', lambda _: common.EMTEST_SKIP_SLOW)(func) + decorated.is_slow = True + return decorated + + +def flaky(note=''): + assert not callable(note) + + if common.EMTEST_SKIP_FLAKY: + return unittest.skip(note) + + if not common.EMTEST_RETRY_FLAKY: + return lambda f: f + + def decorated(func): + @wraps(func) + def modified(self, *args, **kwargs): + # Browser tests have there own method of retrying tests. + if self.is_browser_test(): + self.flaky = True + return func(self, *args, **kwargs) + + for i in range(common.EMTEST_RETRY_FLAKY): + try: + return func(self, *args, **kwargs) + except (AssertionError, subprocess.TimeoutExpired) as exc: + preserved_exc = exc + common.record_flaky_test(self.id(), i, common.EMTEST_RETRY_FLAKY, exc) + + raise AssertionError('Flaky test has failed too many times') from preserved_exc + + return modified + + return decorated + + +def disabled(note=''): + assert not callable(note) + return unittest.skip(note) + + +no_mac = skip_if('no_mac', lambda _: MACOS) + +no_windows = skip_if('no_windows', lambda _: WINDOWS) + +no_wasm64 = skip_if('no_wasm64', lambda t: t.is_wasm64()) + +# 2200mb is the value used by the core_2gb test mode +no_2gb = skip_if('no_2gb', lambda t: t.get_setting('INITIAL_MEMORY') == '2200mb') + +no_4gb = skip_if('no_4gb', lambda t: t.is_4gb()) + +only_windows = skip_if('only_windows', lambda _: not WINDOWS) + +requires_native_clang = skip_if_simple('native clang tests are disabled', lambda _: common.EMTEST_LACKS_NATIVE_CLANG) + +needs_make = skip_if('tool not available on windows bots', lambda _: WINDOWS) + + +def requires_node(func): + assert callable(func) + + @wraps(func) + def decorated(self, *args, **kwargs): + self.require_node() + return func(self, *args, **kwargs) + + return decorated + + +def requires_node_canary(func): + assert callable(func) + + @wraps(func) + def decorated(self, *args, **kwargs): + self.require_node_canary() + return func(self, *args, **kwargs) + + return decorated + + +# Used to mark dependencies in various tests to npm developer dependency +# packages, which might not be installed on Emscripten end users' systems. +def requires_dev_dependency(package): + assert not callable(package) + note = f'requires npm development package "{package}" and EMTEST_SKIP_NODE_DEV_PACKAGES is set' + return skip_if_simple('requires_dev_dependency', lambda _: 'EMTEST_SKIP_NODE_DEV_PACKAGES' in os.environ, note) + + +def requires_wasm64(func): + assert callable(func) + + @wraps(func) + def decorated(self, *args, **kwargs): + self.require_wasm64() + return func(self, *args, **kwargs) + + return decorated + + +def requires_wasm_eh(func): + assert callable(func) + + @wraps(func) + def decorated(self, *args, **kwargs): + self.require_wasm_eh() + return func(self, *args, **kwargs) + + return decorated + + +def requires_v8(func): + assert callable(func) + + @wraps(func) + def decorated(self, *args, **kwargs): + self.require_v8() + return func(self, *args, **kwargs) + + return decorated + + +def requires_wasm2js(func): + assert callable(func) + + @wraps(func) + def decorated(self, *args, **kwargs): + self.require_wasm2js() + return func(self, *args, **kwargs) + + return decorated + + +def requires_jspi(func): + assert callable(func) + + @wraps(func) + def decorated(self, *args, **kwargs): + self.require_jspi() + return func(self, *args, **kwargs) + + return decorated + + +def node_pthreads(func): + assert callable(func) + + @wraps(func) + def decorated(self, *args, **kwargs): + self.setup_node_pthreads() + return func(self, *args, **kwargs) + return decorated + + +def crossplatform(func): + assert callable(func) + func.is_crossplatform_test = True + return func + + +# without EMTEST_ALL_ENGINES set we only run tests in a single VM by +# default. in some tests we know that cross-VM differences may happen and +# so are worth testing, and they should be marked with this decorator +def all_engines(func): + assert callable(func) + + @wraps(func) + def decorated(self, *args, **kwargs): + self.use_all_engines = True + self.set_setting('ENVIRONMENT', 'web,node,shell') + return func(self, *args, **kwargs) + + return decorated + + +# Decorator version of env_modify +def with_env_modify(updates): + assert not callable(updates) + + def decorated(func): + @wraps(func) + def modified(self, *args, **kwargs): + with common.env_modify(updates): + return func(self, *args, **kwargs) + return modified + + return decorated + + +def also_with_wasmfs(func): + assert callable(func) + + @wraps(func) + def metafunc(self, wasmfs, *args, **kwargs): + if DEBUG: + print('parameterize:wasmfs=%d' % wasmfs) + if wasmfs: + self.setup_wasmfs_test() + else: + self.cflags += ['-DMEMFS'] + return func(self, *args, **kwargs) + + parameterize(metafunc, {'': (False,), + 'wasmfs': (True,)}) + return metafunc + + +def also_with_nodefs(func): + @wraps(func) + def metafunc(self, fs, *args, **kwargs): + if DEBUG: + print('parameterize:fs=%s' % (fs)) + if fs == 'nodefs': + self.setup_nodefs_test() + else: + self.cflags += ['-DMEMFS'] + assert fs is None + return func(self, *args, **kwargs) + + parameterize(metafunc, {'': (None,), + 'nodefs': ('nodefs',)}) + return metafunc + + +def also_with_nodefs_both(func): + @wraps(func) + def metafunc(self, fs, *args, **kwargs): + if DEBUG: + print('parameterize:fs=%s' % (fs)) + if fs == 'nodefs': + self.setup_nodefs_test() + elif fs == 'rawfs': + self.setup_noderawfs_test() + else: + self.cflags += ['-DMEMFS'] + assert fs is None + return func(self, *args, **kwargs) + + parameterize(metafunc, {'': (None,), + 'nodefs': ('nodefs',), + 'rawfs': ('rawfs',)}) + return metafunc + + +def with_all_fs(func): + @wraps(func) + def metafunc(self, wasmfs, fs, *args, **kwargs): + if DEBUG: + print('parameterize:fs=%s' % (fs)) + if wasmfs: + self.setup_wasmfs_test() + if fs == 'nodefs': + self.setup_nodefs_test() + elif fs == 'rawfs': + self.setup_noderawfs_test() + else: + self.cflags += ['-DMEMFS'] + assert fs is None + return func(self, *args, **kwargs) + + parameterize(metafunc, {'': (False, None), + 'nodefs': (False, 'nodefs'), + 'rawfs': (False, 'rawfs'), + 'wasmfs': (True, None), + 'wasmfs_nodefs': (True, 'nodefs'), + 'wasmfs_rawfs': (True, 'rawfs')}) + return metafunc + + +def also_with_noderawfs(func): + assert callable(func) + + @wraps(func) + def metafunc(self, rawfs, *args, **kwargs): + if DEBUG: + print('parameterize:rawfs=%d' % rawfs) + if rawfs: + self.setup_noderawfs_test() + else: + self.cflags += ['-DMEMFS'] + return func(self, *args, **kwargs) + + parameterize(metafunc, {'': (False,), + 'rawfs': (True,)}) + return metafunc + + +def also_with_minimal_runtime(func): + assert callable(func) + + @wraps(func) + def metafunc(self, with_minimal_runtime, *args, **kwargs): + if DEBUG: + print('parameterize:minimal_runtime=%s' % with_minimal_runtime) + if self.get_setting('MINIMAL_RUNTIME'): + self.skipTest('MINIMAL_RUNTIME already enabled in test config') + if with_minimal_runtime: + if self.get_setting('MODULARIZE') == 'instance' or self.get_setting('WASM_ESM_INTEGRATION'): + self.skipTest('MODULARIZE=instance is not compatible with MINIMAL_RUNTIME') + self.set_setting('MINIMAL_RUNTIME', 1) + # This extra helper code is needed to cleanly handle calls to exit() which throw + # an ExitCode exception. + self.cflags += ['--pre-js', test_file('minimal_runtime_exit_handling.js')] + return func(self, *args, **kwargs) + + parameterize(metafunc, {'': (False,), + 'minimal_runtime': (True,)}) + return metafunc + + +def also_without_bigint(func): + assert callable(func) + + @wraps(func) + def metafunc(self, no_bigint, *args, **kwargs): + if DEBUG: + print('parameterize:no_bigint=%s' % no_bigint) + if no_bigint: + if self.get_setting('WASM_BIGINT') is not None: + self.skipTest('redundant in bigint test config') + self.set_setting('WASM_BIGINT', 0) + return func(self, *args, **kwargs) + + parameterize(metafunc, {'': (False,), + 'no_bigint': (True,)}) + return metafunc + + +def also_with_wasm64(func): + assert callable(func) + + @wraps(func) + def metafunc(self, with_wasm64, *args, **kwargs): + if DEBUG: + print('parameterize:wasm64=%s' % with_wasm64) + if with_wasm64: + self.require_wasm64() + self.set_setting('MEMORY64') + return func(self, *args, **kwargs) + + parameterize(metafunc, {'': (False,), + 'wasm64': (True,)}) + return metafunc + + +def also_with_wasm2js(func): + assert callable(func) + + @wraps(func) + def metafunc(self, with_wasm2js, *args, **kwargs): + assert self.get_setting('WASM') is None + if DEBUG: + print('parameterize:wasm2js=%s' % with_wasm2js) + if with_wasm2js: + self.require_wasm2js() + self.set_setting('WASM', 0) + return func(self, *args, **kwargs) + + parameterize(metafunc, {'': (False,), + 'wasm2js': (True,)}) + return metafunc + + +def can_do_standalone(self, impure=False): + # Pure standalone engines don't support MEMORY64 yet. Even with MEMORY64=2 (lowered) + # the WASI APIs that take pointer values don't have 64-bit variants yet. + if not impure: + if self.get_setting('MEMORY64'): + return False + # This is way to detect the core_2gb test mode in test_core.py + if self.get_setting('INITIAL_MEMORY') == '2200mb': + return False + return self.is_wasm() and \ + self.get_setting('STACK_OVERFLOW_CHECK', 0) < 2 and \ + not self.get_setting('MINIMAL_RUNTIME') and \ + not self.get_setting('WASM_ESM_INTEGRATION') and \ + not self.get_setting('SAFE_HEAP') and \ + not any(a.startswith('-fsanitize=') for a in self.cflags) + + +# Impure means a test that cannot run in a wasm VM yet, as it is not 100% +# standalone. We can still run them with the JS code though. +def also_with_standalone_wasm(impure=False): + def decorated(func): + @wraps(func) + def metafunc(self, standalone, *args, **kwargs): + if DEBUG: + print('parameterize:standalone=%s' % standalone) + if standalone: + if not can_do_standalone(self, impure): + self.skipTest('Test configuration is not compatible with STANDALONE_WASM') + self.set_setting('STANDALONE_WASM') + if not impure: + self.set_setting('PURE_WASI') + self.cflags.append('-Wno-unused-command-line-argument') + # if we are impure, disallow all wasm engines + if impure: + self.wasm_engines = [] + return func(self, *args, **kwargs) + + parameterize(metafunc, {'': (False,), + 'standalone': (True,)}) + return metafunc + + return decorated + + +def also_with_asan(func): + assert callable(func) + + @wraps(func) + def metafunc(self, asan, *args, **kwargs): + if asan: + if self.is_wasm64(): + self.skipTest('TODO: ASAN in memory64') + if self.is_2gb() or self.is_4gb(): + self.skipTest('asan doesnt support GLOBAL_BASE') + self.cflags.append('-fsanitize=address') + return func(self, *args, **kwargs) + + parameterize(metafunc, {'': (False,), + 'asan': (True,)}) + return metafunc + + +def also_with_modularize(func): + assert callable(func) + + @wraps(func) + def metafunc(self, modularize, *args, **kwargs): + if modularize: + if self.get_setting('DECLARE_ASM_MODULE_EXPORTS') == 0: + self.skipTest('DECLARE_ASM_MODULE_EXPORTS=0 is not compatible with MODULARIZE') + if self.get_setting('STRICT_JS'): + self.skipTest('MODULARIZE is not compatible with STRICT_JS') + if self.get_setting('WASM_ESM_INTEGRATION'): + self.skipTest('MODULARIZE is not compatible with WASM_ESM_INTEGRATION') + self.cflags += ['--extern-post-js', test_file('modularize_post_js.js'), '-sMODULARIZE'] + return func(self, *args, **kwargs) + + parameterize(metafunc, {'': (False,), + 'modularize': (True,)}) + return metafunc + + +# Tests exception handling and setjmp/longjmp handling. This tests three +# combinations: +# - Emscripten EH + Emscripten SjLj +# - Wasm EH + Wasm SjLj +# - Wasm EH + Wasm SjLj (Legacy) +def with_all_eh_sjlj(func): + assert callable(func) + + @wraps(func) + def metafunc(self, mode, *args, **kwargs): + if DEBUG: + print('parameterize:eh_mode=%s' % mode) + if mode in {'wasm', 'wasm_legacy'}: + # Wasm EH is currently supported only in wasm backend and V8 + if self.is_wasm2js(): + self.skipTest('wasm2js does not support wasm EH/SjLj') + self.cflags.append('-fwasm-exceptions') + self.set_setting('SUPPORT_LONGJMP', 'wasm') + if mode == 'wasm': + self.require_wasm_eh() + if mode == 'wasm_legacy': + self.require_wasm_legacy_eh() + else: + self.set_setting('DISABLE_EXCEPTION_CATCHING', 0) + self.set_setting('SUPPORT_LONGJMP', 'emscripten') + # DISABLE_EXCEPTION_CATCHING=0 exports __cxa_can_catch, + # so if we don't build in C++ mode, wasm-ld will + # error out because libc++abi is not included. See + # https://github.com/emscripten-core/emscripten/pull/14192 for details. + self.set_setting('DEFAULT_TO_CXX') + return func(self, *args, **kwargs) + + parameterize(metafunc, {'emscripten': ('emscripten',), + 'wasm': ('wasm',), + 'wasm_legacy': ('wasm_legacy',)}) + return metafunc + + +# This works just like `with_all_eh_sjlj` above but doesn't enable exceptions. +# Use this for tests that use setjmp/longjmp but not exceptions handling. +def with_all_sjlj(func): + assert callable(func) + + @wraps(func) + def metafunc(self, mode, *args, **kwargs): + if mode in {'wasm', 'wasm_legacy'}: + if self.is_wasm2js(): + self.skipTest('wasm2js does not support wasm SjLj') + self.set_setting('SUPPORT_LONGJMP', 'wasm') + if mode == 'wasm': + self.require_wasm_eh() + if mode == 'wasm_legacy': + self.require_wasm_legacy_eh() + else: + self.set_setting('SUPPORT_LONGJMP', 'emscripten') + return func(self, *args, **kwargs) + + parameterize(metafunc, {'emscripten': ('emscripten',), + 'wasm': ('wasm',), + 'wasm_legacy': ('wasm_legacy',)}) + return metafunc + + +def parameterize(func, parameters): + """Add additional parameterization to a test function. + + This function create or adds to the `_parameterize` property of a function + which is then expanded by the RunnerMeta metaclass into multiple separate + test functions. + """ + prev = getattr(func, '_parameterize', None) + assert not any(p.startswith('_') for p in parameters) + if prev: + # If we're parameterizing 2nd time, construct a cartesian product for various combinations. + func._parameterize = { + '_'.join(filter(None, [k1, k2])): v2 + v1 for (k1, v1), (k2, v2) in itertools.product(prev.items(), parameters.items()) + } + else: + func._parameterize = parameters + + +def parameterized(parameters): + """ + Mark a test as parameterized. + + Usage: + @parameterized({ + 'subtest1': (1, 2, 3), + 'subtest2': (4, 5, 6), + }) + def test_something(self, a, b, c): + ... # actual test body + + This is equivalent to defining two tests: + + def test_something_subtest1(self): + # runs test_something(1, 2, 3) + + def test_something_subtest2(self): + # runs test_something(4, 5, 6) + """ + def decorator(func): + parameterize(func, parameters) + return func + return decorator diff --git a/test/test_benchmark.py b/test/test_benchmark.py index 7a158f6875b22..993fd4f000d5a 100644 --- a/test/test_benchmark.py +++ b/test/test_benchmark.py @@ -21,7 +21,8 @@ import clang_native import common import jsrun -from common import needs_make, read_binary, read_file, test_file +from common import read_binary, read_file, test_file +from decorators import needs_make from tools import building, utils from tools.shared import CLANG_CC, CLANG_CXX, EMCC, PIPE, config, run_process diff --git a/test/test_browser.py b/test/test_browser.py index 1ac4a75ec5c66..e808d1a8a96cf 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -33,32 +33,34 @@ HttpServerThread, Reporting, RunnerCore, - also_with_asan, - also_with_minimal_runtime, - also_with_wasm2js, - also_with_wasmfs, copytree, create_file, - disabled, ensure_dir, find_browser_test_file, - flaky, has_browser, is_chrome, is_firefox, is_safari, + path_from_root, + read_file, + test_file, +) +from decorators import ( + also_with_asan, + also_with_minimal_runtime, + also_with_wasm2js, + also_with_wasmfs, + disabled, + flaky, no_2gb, no_4gb, no_wasm64, parameterize, parameterized, - path_from_root, - read_file, requires_dev_dependency, requires_wasm2js, skip_if, skip_if_simple, - test_file, with_all_sjlj, ) diff --git a/test/test_codesize.py b/test/test_codesize.py index 695c1d869f300..4316e17cc537d 100644 --- a/test/test_codesize.py +++ b/test/test_codesize.py @@ -14,12 +14,11 @@ RunnerCore, compiler_for, create_file, - node_pthreads, - parameterized, read_binary, read_file, test_file, ) +from decorators import node_pthreads, parameterized from tools import building, shared diff --git a/test/test_core.py b/test/test_core.py index 0356186905712..1e394f774fe3e 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -26,6 +26,15 @@ PYTHON, WEBIDL_BINDER, RunnerCore, + compiler_for, + create_file, + env_modify, + path_from_root, + read_binary, + read_file, + test_file, +) +from decorators import ( all_engines, also_with_minimal_runtime, also_with_modularize, @@ -36,11 +45,8 @@ also_with_wasmfs, also_without_bigint, can_do_standalone, - compiler_for, - create_file, crossplatform, disabled, - env_modify, flaky, is_slow_test, needs_make, @@ -51,9 +57,6 @@ node_pthreads, parameterize, parameterized, - path_from_root, - read_binary, - read_file, requires_dev_dependency, requires_jspi, requires_native_clang, @@ -63,7 +66,6 @@ requires_wasm2js, requires_wasm_eh, skip_if, - test_file, with_all_eh_sjlj, with_all_fs, with_all_sjlj, diff --git a/test/test_interactive.py b/test/test_interactive.py index 22ecb2789ab98..4cf0a51479648 100644 --- a/test/test_interactive.py +++ b/test/test_interactive.py @@ -10,13 +10,8 @@ if __name__ == '__main__': raise Exception('do not run this file directly; do something like: test/runner.py interactive') -from common import ( - BrowserCore, - also_with_minimal_runtime, - create_file, - parameterized, - test_file, -) +from common import BrowserCore, create_file, test_file +from decorators import also_with_minimal_runtime, parameterized from tools.shared import WINDOWS diff --git a/test/test_jslib.py b/test/test_jslib.py index 9e5bef1c277ce..2c0f78b6652ba 100644 --- a/test/test_jslib.py +++ b/test/test_jslib.py @@ -6,13 +6,8 @@ import shutil from subprocess import PIPE -from common import ( - RunnerCore, - create_file, - parameterized, - read_file, - test_file, -) +from common import RunnerCore, create_file, read_file, test_file +from decorators import parameterized from tools.shared import EMCC from tools.utils import delete_file diff --git a/test/test_other.py b/test/test_other.py index 22b2861d27835..fb48932a158c8 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -42,6 +42,15 @@ TEST_ROOT, WEBIDL_BINDER, RunnerCore, + copytree, + create_file, + ensure_dir, + env_modify, + make_executable, + path_from_root, + test_file, +) +from decorators import ( all_engines, also_with_asan, also_with_minimal_runtime, @@ -52,22 +61,16 @@ also_with_wasm64, also_with_wasmfs, also_without_bigint, - copytree, - create_file, crossplatform, disabled, - ensure_dir, - env_modify, flaky, is_slow_test, - make_executable, no_mac, no_windows, node_pthreads, only_windows, parameterize, parameterized, - path_from_root, requires_dev_dependency, requires_jspi, requires_native_clang, @@ -77,7 +80,6 @@ requires_v8, requires_wasm64, requires_wasm_eh, - test_file, with_all_eh_sjlj, with_all_fs, with_all_sjlj, diff --git a/test/test_posixtest.py b/test/test_posixtest.py index 3d420f58cc9ff..ca3e721ca5ab5 100644 --- a/test/test_posixtest.py +++ b/test/test_posixtest.py @@ -14,7 +14,8 @@ import unittest import test_posixtest_browser -from common import RunnerCore, node_pthreads, path_from_root +from common import RunnerCore, path_from_root +from decorators import node_pthreads testsuite_root = path_from_root('test/third_party/posixtestsuite') diff --git a/test/test_sanity.py b/test/test_sanity.py index 75dd4c2476a26..111661c0da3c0 100644 --- a/test/test_sanity.py +++ b/test/test_sanity.py @@ -19,15 +19,13 @@ EMBUILDER, RunnerCore, create_file, - crossplatform, ensure_dir, env_modify, make_executable, - parameterized, path_from_root, test_file, - with_env_modify, ) +from decorators import crossplatform, parameterized, with_env_modify from tools import cache, ports, response_file, shared, utils from tools.config import EM_CONFIG diff --git a/test/test_sockets.py b/test/test_sockets.py index 82c769b1d42e6..61381d38e31e6 100644 --- a/test/test_sockets.py +++ b/test/test_sockets.py @@ -17,15 +17,11 @@ import clang_native import common -from common import ( - NON_ZERO, - PYTHON, - BrowserCore, - create_file, +from common import NON_ZERO, PYTHON, BrowserCore, create_file, read_file +from decorators import ( crossplatform, no_windows, parameterized, - read_file, requires_dev_dependency, requires_native_clang, test_file, diff --git a/test/test_stress.py b/test/test_stress.py index 7ce8eb750a141..ae4731efbf413 100644 --- a/test/test_stress.py +++ b/test/test_stress.py @@ -16,13 +16,8 @@ import subprocess import threading -from common import ( - NON_ZERO, - RunnerCore, - also_with_modularize, - is_slow_test, - node_pthreads, -) +from common import NON_ZERO, RunnerCore +from decorators import also_with_modularize, is_slow_test, node_pthreads class stress(RunnerCore):