Fuzzer: Exit with a different returncode for comparison errors

scripts/fuzz_opt.py

@@ -2611,7 +2611,7 @@ def get_random_opts():
         counter += 1
         if given_seed is not None:
             seed = given_seed
-            given_seed_passed = True
+            given_seed_error = 0
         else:
             seed = random.randint(0, 1 << 64)
         random.seed(seed)
@@ -2652,10 +2652,16 @@ def get_random_opts():
             traceback.print_tb(tb)
             print('-----------------------------------------')
             print('!')
+            # Default to an error code of 1, but change it for certain errors,
+            # so we report them differently (useful for the reducer to keep
+            # reducing the exact same error category)
+            if given_seed is not None:
+                given_seed_error = 1
             for arg in e.args:
                 print(arg)
-            if given_seed is not None:
-                given_seed_passed = False
+                if type(arg) is str:
+                    if 'comparison error' in arg:
+                        given_seed_error = 2
 
             # We want to generate a template reducer script only when there is
             # no given wasm file. That we have a given wasm file means we are no
@@ -2700,7 +2706,7 @@ def get_random_opts():
 %(wasm_opt)s %(features)s %(temp_wasm)s
 echo "  " $?
 
-echo "The following value should be 1:"
+echo "The following value should be >0:"
 
 if [ -z "$BINARYEN_FIRST_WASM" ]; then
   # run the command normally
@@ -2778,7 +2784,7 @@ def get_random_opts():
 
 The following value should be 0:
   0
-The following value should be 1:
+The following value should be >0:
   1
 
 (If it does not, then one possible issue is that the fuzzer fails to write a
@@ -2809,9 +2815,9 @@ def get_random_opts():
             print('  ', testcase_handler.__class__.__name__ + ':', testcase_handler.count_runs())
 
     if given_seed is not None:
-        if given_seed_passed:
+        if not given_seed_error:
             print('(finished running seed %d without error)' % given_seed)
             sys.exit(0)
         else:
             print('(finished running seed %d, see error above)' % given_seed)
-            sys.exit(1)
+            sys.exit(given_seed_error)