From 2bd696c9a5c76e5ef3c024e7fcc84b8a9095876b Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 30 Oct 2025 17:01:54 -0700 Subject: [PATCH 1/4] fix --- scripts/fuzz_opt.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py index 9a834e7647d..76a2e31f5fb 100755 --- a/scripts/fuzz_opt.py +++ b/scripts/fuzz_opt.py @@ -455,12 +455,17 @@ def compare(x, y, context, verbose=True): if x != y and x != IGNORE and y != IGNORE: message = ''.join([a + '\n' for a in difflib.unified_diff(x.splitlines(), y.splitlines(), fromfile='expected', tofile='actual')]) if verbose: - raise Exception(context + " comparison error, expected to have '%s' == '%s', diff:\n\n%s" % ( + print(context + " comparison error, expected to have '%s' == '%s', diff:\n\n%s" % ( x, y, message )) else: - raise Exception(context + "\nDiff:\n\n%s" % (message)) + print(context + "\nDiff:\n\n%s" % (message)) + traceback.print_stack() + # Exit with code 2, which is different than code 1 that happens for a + # thrown Python exception. This allows the reducer to not get confused + # between a comparison error and some other kind of error. + sys.exit(2) # converts a possibly-signed integer to an unsigned integer From ec81e4d9c8277f66487d9501e2f40c9c7b593be9 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Nov 2025 08:54:14 -0800 Subject: [PATCH 2/4] fix --- scripts/fuzz_opt.py | 132 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 110 insertions(+), 22 deletions(-) diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py index b7a7f1842c0..0cbb785e7ca 100755 --- a/scripts/fuzz_opt.py +++ b/scripts/fuzz_opt.py @@ -455,17 +455,12 @@ def compare(x, y, context, verbose=True): if x != y and x != IGNORE and y != IGNORE: message = ''.join([a + '\n' for a in difflib.unified_diff(x.splitlines(), y.splitlines(), fromfile='expected', tofile='actual')]) if verbose: - print(context + " comparison error, expected to have '%s' == '%s', diff:\n\n%s" % ( + raise Exception(context + " comparison error, expected to have '%s' == '%s', diff:\n\n%s" % ( x, y, message )) else: - print(context + "\nDiff:\n\n%s" % (message)) - traceback.print_stack() - # Exit with code 2, which is different than code 1 that happens for a - # thrown Python exception. This allows the reducer to not get confused - # between a comparison error and some other kind of error. - sys.exit(2) + raise Exception(context + "\nDiff:\n\n%s" % (message)) # converts a possibly-signed integer to an unsigned integer @@ -1788,9 +1783,9 @@ def ensure(self): tar.close() -# Tests linking two wasm files at runtime, and that optimizations do not break -# anything. This is similar to Split(), but rather than split a wasm file into -# two and link them at runtime, this starts with two separate wasm files. +# Generates two wasm and tests interesting interactions between them. This is a +# little similar to Split(), but rather than split one wasm file into two and +# test that, we start with two. # # Fuzzing failures here is a little trickier, as there are two wasm files. # You can reduce the primary file by finding the secondary one in the log @@ -1817,7 +1812,7 @@ def ensure(self): class Two(TestCaseHandler): # Run at relatively high priority, as this is the main place we check cross- # module interactions. - frequency = 1 + frequency = 1 # TODO: We may want even higher priority here def handle(self, wasm): # Generate a second wasm file. (For fuzzing, we may be given one, but we @@ -1870,9 +1865,50 @@ def handle(self, wasm): print(f'warning: no calls in output. output:\n{output}') assert calls_in_output == len(exports), exports + # Merge the files and run them that way. The result should be the same, + # even if we optimize. TODO: merge (no pun intended) the rest of Merge + # into here. + merged = abspath('merged.wasm') + run([in_bin('wasm-merge'), wasm, 'primary', second_wasm, 'secondary', + '-o', merged, '--rename-export-conflicts', '-all']) + + # Usually also optimize the merged module. Optimizations are very + # interesting here, because after merging we can safely do even closed- + # world optimizations, making very aggressive changes that should still + # behave the same as before merging. + if random.random() < 0.8: + merged_opt = abspath('merged.opt.wasm') + opts = get_random_opts() + run([in_bin('wasm-opt'), merged, '-o', merged_opt, '-all'] + opts) + merged = merged_opt + + if not wasm_notices_export_changes(merged): + # wasm-merge combines exports, which can alter their indexes and + # lead to noticeable differences if the wasm is sensitive to such + # things. We only compare the output if that is not an issue. + merged_output = run_bynterp(merged, args=['--fuzz-exec-before', '-all']) + + if merged_output == IGNORE: + # The original output was ok, but after merging it becomes + # something we must ignore. This can happen when we optimize, if + # the optimizer reorders a normal trap (say a null exception) + # with a host limit trap (say an allocation limit). Nothing to + # do here, but verify we did optimize, as otherwise this is + # inexplicable. + assert merged == abspath('merged.opt.wasm') + else: + self.compare_to_merged_output(output, merged_output) + + # The rest of the testing here depends on being to optimize the + # two modules independently, which closed-world can break. + if CLOSED_WORLD: + return + + # Fix up the normal output for later comparisons. output = fix_output(output) - # Optimize at least one of the two. + # We can optimize and compare the results. Optimize at least one of + # the two. wasms = [wasm, second_wasm] for i in range(random.randint(1, 2)): wasm_index = random.randint(0, 1) @@ -1886,7 +1922,7 @@ def handle(self, wasm): optimized_output = run_bynterp(wasms[0], args=['--fuzz-exec-before', f'--fuzz-exec-second={wasms[1]}']) optimized_output = fix_output(optimized_output) - compare(output, optimized_output, 'Two') + compare(output, optimized_output, 'Two-Opt') # If we can, also test in V8. We also cannot compare if there are NaNs # (as optimizations can lead to different outputs), and we must @@ -1912,10 +1948,56 @@ def handle(self, wasm): compare(output, optimized_output, 'Two-V8') - def can_run_on_wasm(self, wasm): - # We cannot optimize wasm files we are going to link in closed world - # mode. - return not CLOSED_WORLD + def compare_to_merged_output(self, output, merged_output): + # Comparing the original output from two files to the output after + # merging them is not trivial. First, remove the extra logging that + # --fuzz-exec-second adds. + output = output.replace('[fuzz-exec] running second module\n', '') + + # Fix up both outputs. + output = fix_output(output) + merged_output = fix_output(merged_output) + + # Finally, align the export names. We merged with + # --rename-export-conflicts, so that all exports remain exported, + # allowing a full comparison, but we do need to handle the different + # names. We do so by matching the export names in the logging. + output_lines = output.splitlines() + merged_output_lines = merged_output.splitlines() + + if len(output_lines) != len(merged_output_lines): + # The line counts don't even match. Just compare them, which will + # emit a nice error for that. + compare(output, merged_output, 'Two-Counts') + assert False, 'we should have errored on the line counts' + + for i in range(len(output_lines)): + a = output_lines[i] + b = merged_output_lines[i] + if a == b: + continue + if a.startswith(FUZZ_EXEC_CALL_PREFIX): + # Fix up + # [fuzz-exec] calling foo/bar + # for different foo/bar. Just copy the original. + assert b.startswith(FUZZ_EXEC_CALL_PREFIX) + merged_output_lines[i] = output_lines[i] + elif a.startswith(FUZZ_EXEC_NOTE_RESULT): + # Fix up + # [fuzz-exec] note result: foo/bar => 42 + # for different foo/bar. We do not want to copy the result here, + # which might differ (that would be a bug we want to find). + assert b.startswith(FUZZ_EXEC_NOTE_RESULT) + assert a.count(' => ') == 1 + assert b.count(' => ') == 1 + a_prefix, a_result = a.split(' => ') + b_prefix, b_result = b.split(' => ') + # Copy a's prefix with b's result. + merged_output_lines[i] = a_prefix + ' => ' + b_result + + merged_output = '\n'.join(merged_output_lines) + + compare(output, merged_output, 'Two-Merged') # Test --fuzz-preserve-imports-exports, which never modifies imports or exports. @@ -2529,7 +2611,7 @@ def get_random_opts(): counter += 1 if given_seed is not None: seed = given_seed - given_seed_passed = True + given_seed_error = 0 else: seed = random.randint(0, 1 << 64) random.seed(seed) @@ -2570,10 +2652,16 @@ def get_random_opts(): traceback.print_tb(tb) print('-----------------------------------------') print('!') + # Default to an error code of 1, but change it for certain errors, + # so we report them differently (useful for the reducer to keep + # reducing the exact same error category) + if given_seed is not None: + given_seed_error = 1 for arg in e.args: print(arg) - if given_seed is not None: - given_seed_passed = False + if type(arg) is str: + if 'comparison error' in arg: + given_seen_error = 2 # We want to generate a template reducer script only when there is # no given wasm file. That we have a given wasm file means we are no @@ -2727,9 +2815,9 @@ def get_random_opts(): print(' ', testcase_handler.__class__.__name__ + ':', testcase_handler.count_runs()) if given_seed is not None: - if given_seed_passed: + if not given_seed_error: print('(finished running seed %d without error)' % given_seed) sys.exit(0) else: print('(finished running seed %d, see error above)' % given_seed) - sys.exit(1) + sys.exit(given_seen_error) From 16a451b7631583c89e373dbd3a3b8e28886bf359 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Nov 2025 08:55:51 -0800 Subject: [PATCH 3/4] fix --- scripts/fuzz_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py index 0cbb785e7ca..cdaa3c4561d 100755 --- a/scripts/fuzz_opt.py +++ b/scripts/fuzz_opt.py @@ -2661,7 +2661,7 @@ def get_random_opts(): print(arg) if type(arg) is str: if 'comparison error' in arg: - given_seen_error = 2 + given_seed_error = 2 # We want to generate a template reducer script only when there is # no given wasm file. That we have a given wasm file means we are no @@ -2820,4 +2820,4 @@ def get_random_opts(): sys.exit(0) else: print('(finished running seed %d, see error above)' % given_seed) - sys.exit(given_seen_error) + sys.exit(given_seed_error) From 065b6bc28fff6aa606488176db0107077738e9a1 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 7 Nov 2025 09:00:07 -0800 Subject: [PATCH 4/4] fix --- scripts/fuzz_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py index cdaa3c4561d..51b16f6b3c5 100755 --- a/scripts/fuzz_opt.py +++ b/scripts/fuzz_opt.py @@ -2706,7 +2706,7 @@ def get_random_opts(): %(wasm_opt)s %(features)s %(temp_wasm)s echo " " $? -echo "The following value should be 1:" +echo "The following value should be >0:" if [ -z "$BINARYEN_FIRST_WASM" ]; then # run the command normally @@ -2784,7 +2784,7 @@ def get_random_opts(): The following value should be 0: 0 -The following value should be 1: +The following value should be >0: 1 (If it does not, then one possible issue is that the fuzzer fails to write a