diff --git a/tests/compiler.js b/tests/compiler.js
index 863787a533..8e3764912b 100644
--- a/tests/compiler.js
+++ b/tests/compiler.js
@@ -321,6 +321,10 @@ async function runTest(basename) {
       }
       compareStderr.end(SUCCESS);
       return prepareResult(SUCCESS);
+    } else if (error) {
+      // Don't bother comparing fixtures or doing anything else if the
+      // compilation failed.
+      return prepareResult(FAILURE, "compile failed");
     }
 
     const afterCompileResult = afterCompile("debug");