chore: Improve external parser tests

Makefile

@@ -150,7 +150,8 @@ test-ci-coverage:
 bootstrap-flow:
 	rm -rf build/flow
 	mkdir -p build
-	git clone --single-branch --shallow-since=2021-05-01 https://github.com/facebook/flow.git build/flow
+	git clone --filter=blob:none --sparse --single-branch --shallow-since=2021-05-01 https://github.com/facebook/flow.git build/flow
+	cd build/flow && git sparse-checkout set "src/parser/test/flow"
 	cd build/flow && git checkout -q $(FLOW_COMMIT)
 
 test-flow:
@@ -162,7 +163,8 @@ test-flow-update-allowlist:
 bootstrap-typescript:
 	rm -rf ./build/typescript
 	mkdir -p ./build
-	git clone --single-branch --shallow-since=2022-04-01 https://github.com/microsoft/TypeScript.git ./build/typescript
+	git clone --filter=blob:none --sparse --single-branch --shallow-since=2022-04-01 https://github.com/microsoft/TypeScript.git ./build/typescript
+	cd build/typescript && git sparse-checkout set "tests"
 	cd build/typescript && git checkout -q $(TYPESCRIPT_COMMIT)
 
 test-typescript:
@@ -174,7 +176,8 @@ test-typescript-update-allowlist:
 bootstrap-test262:
 	rm -rf build/test262
 	mkdir -p build
-	git clone --single-branch --shallow-since=2021-05-01 https://github.com/tc39/test262.git build/test262
+	git clone --filter=blob:none --sparse --single-branch --shallow-since=2021-05-01 https://github.com/tc39/test262.git build/test262
+	cd build/test262 && git sparse-checkout set "test"
 	cd build/test262 && git checkout -q $(TEST262_COMMIT)
 
 test-test262:

scripts/parser-tests/test262/allowlist.txt

@@ -1,6 +1,12 @@
-language/expressions/class/decorator/syntax/class-valid/decorator-member-expr-private-identifier.js(default)
-language/expressions/class/decorator/syntax/class-valid/decorator-member-expr-private-identifier.js(strict mode)
+# 2 invalid programs did not produce a parsing error
+
 language/import/json-invalid.js(default)
 language/import/json-invalid.js(strict mode)
+
+
+# 4 valid programs produced a parsing error
+
+language/expressions/class/decorator/syntax/class-valid/decorator-member-expr-private-identifier.js(default)
+language/expressions/class/decorator/syntax/class-valid/decorator-member-expr-private-identifier.js(strict mode)
 language/statements/class/decorator/syntax/class-valid/decorator-member-expr-private-identifier.js(default)
 language/statements/class/decorator/syntax/class-valid/decorator-member-expr-private-identifier.js(strict mode)

scripts/parser-tests/test262/index.js

@@ -167,7 +167,7 @@ function* getPlugins(features) {
 }
 
 const runner = new TestRunner({
-  testDir: path.join(dirname, "../../../build/test262"),
+  testDir: path.join(dirname, "../../../build/test262").replace(/\\/g, "/"),
   allowlist: path.join(dirname, "allowlist.txt"),
   logInterval: 500,
   shouldUpdate: process.argv.includes("--update-allowlist"),
@@ -179,7 +179,7 @@ const runner = new TestRunner({
 
     for await (const test of stream) {
       // strip test/
-      const fileName = test.file.slice(5);
+      const fileName = test.file.slice(5).replace(/\\/g, "/");
 
       if (ignoredTests.some(start => fileName.startsWith(start))) continue;
 

scripts/parser-tests/typescript/allowlist.txt

@@ -1,3 +1,18 @@
+# 9 invalid programs did not produce a parsing error
+
+bigintIndex.ts
+classExpressionWithDecorator1.ts
+collisionArgumentsArrowFunctions.ts # TypeScript doesn't allow a parameter to be named arguments even in non-strict mode, which we don't catch.
+collisionArgumentsFunction.ts # TypeScript doesn't allow a parameter to be named arguments even in non-strict mode, which we don't catch.
+collisionArgumentsFunctionExpressions.ts # TypeScript doesn't allow a parameter to be named arguments even in non-strict mode, which we don't catch.
+dynamicImportTrailingComma.ts
+es3-oldStyleOctalLiteralInEnums.ts # We don't support ES3-style octal literal errors.
+exportDeclarationsInAmbientNamespaces2.ts
+multipleExports.ts
+
+
+# 143 valid programs produced a parsing error
+
 ArrowFunctionExpression1.ts
 MemberAccessorDeclaration15.ts
 ParameterList13.ts
@@ -17,15 +32,10 @@ augmentedTypesEnum2.ts
 augmentedTypesFunction.ts
 augmentedTypesInterface.ts
 augmentedTypesVar.ts
-bigintIndex.ts
 classCannotExtendVar.ts
-classExpressionWithDecorator1.ts
 classExtendsMultipleBaseClasses.ts
 classOverloadForFunction.ts
 classWithEmptyTypeParameter.ts
-collisionArgumentsArrowFunctions.ts # TypeScript doesn't allow a parameter to be named arguments even in non-strict mode, which we don't catch.
-collisionArgumentsFunction.ts # TypeScript doesn't allow a parameter to be named arguments even in non-strict mode, which we don't catch.
-collisionArgumentsFunctionExpressions.ts # TypeScript doesn't allow a parameter to be named arguments even in non-strict mode, which we don't catch.
 constDeclarations-invalidContexts.ts
 constDeclarations-scopes.ts
 constDeclarations-validContexts.ts
@@ -49,12 +59,10 @@ duplicateLabel2.ts
 duplicatePackage_withErrors.ts
 duplicateVarAndImport.ts
 duplicateVarAndImport2.ts
-dynamicImportTrailingComma.ts
 emptyGenericParamList.ts
 emptyTypeArgumentList.ts
 emptyTypeArgumentListWithNew.ts
 enumGenericTypeClash.ts
-es3-oldStyleOctalLiteralInEnums.ts # We don't support ES3-style octal literal errors.
 es3-oldStyleOctalLiteralTypes.ts
 es6ImportDefaultBindingFollowedWithNamedImport1InEs5.ts
 es6ImportDefaultBindingMergeErrors.ts
@@ -67,7 +75,6 @@ exportAssignmentWithDeclareModifier.ts
 exportAssignmentWithExportModifier.ts
 exportClassWithoutName.ts
 exportDeclarationsInAmbientNamespaces.ts
-exportDeclarationsInAmbientNamespaces2.ts
 exportDefaultAsyncFunction2.ts
 exportEqualsOfModule.ts
 exportInterfaceClassAndValue.ts
@@ -124,7 +131,6 @@ moduleDuplicateIdentifiers.ts
 moduleSharesNameWithImportDeclarationInsideIt3.ts
 moduleSharesNameWithImportDeclarationInsideIt5.ts
 multipleClassPropertyModifiersErrors.ts
-multipleExports.ts
 multipleInheritance.ts
 nameCollisions.ts
 noImplicitAnyDestructuringVarDeclaration.ts

scripts/parser-tests/utils/parser-test-runner.js

@@ -50,6 +50,7 @@ class TestRunner {
       test.actualError = false;
     } catch (err) {
       test.actualError = true;
+      test.actualErrorObject = err;
     }
 
     test.result = test.expectedError !== test.actualError ? "fail" : "pass";
@@ -85,20 +86,61 @@ class TestRunner {
       .map(test => test.id)
       .concat(summary.unrecognized);
 
-    const updated = summary.disallowed.falsePositive
-      .concat(summary.disallowed.falseNegative)
-      .map(test => test.id);
+    const allowedFalsePositiveIds = summary.allowed.falsePositive.map(
+      test => test.id
+    );
+
+    let invalidWithoutError = [];
+    let validWithError = [];
 
     for (const line of contents.split("\n")) {
       const testId = line.replace(/#.*$/, "").trim();
-      if (!toRemove.includes(testId) && line) {
-        updated.push(line);
+      if (testId && !toRemove.includes(testId)) {
+        if (allowedFalsePositiveIds.includes(testId)) {
+          invalidWithoutError.push(line);
+        } else {
+          validWithError.push(line);
+        }
       }
     }
 
-    updated.sort();
+    invalidWithoutError = invalidWithoutError.concat(
+      summary.disallowed.falsePositive.map(test => test.id)
+    );
+    validWithError = validWithError.concat(
+      summary.disallowed.falseNegative.map(test => test.id)
+    );
+
+    invalidWithoutError.sort();
+    validWithError.sort();
+
+    const errorsMap = new Map();
+    summary.allowed.falseNegative
+      .concat(summary.disallowed.falseNegative)
+      .forEach(test => {
+        errorsMap.set(test.id, test.actualErrorObject);
+      });
+
+    const updated = [
+      `# ${invalidWithoutError.length} invalid programs did not produce a parsing error\n`,
+      "\n",
+      invalidWithoutError.join("\n"),
+      "\n",
+      "\n",
+      "\n",
+      `# ${validWithError.length} valid programs produced a parsing error\n`,
+      "\n",
+      ...validWithError
+        // .map(
+        //   v =>
+        //     `${v}\n# ${JSON.stringify(errorsMap.get(v), null, 2)
+        //       .trimEnd()
+        //       .replace(/\n/g, "\n#")}\n`
+        // )
+        .join("\n"),
+    ];
 
-    await fs.writeFile(this.allowlist, updated.join("\n") + "\n", "utf8");
+    await fs.writeFile(this.allowlist, updated.join("") + "\n", "utf8");
   }
 
   interpret(results, allowlist) {