update runner

babel · Sep 5, 2022 · 46824f3 · 46824f3
1 parent 14ea25f
commit 46824f3
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 10 deletions.
diff --git a/Makefile b/Makefile
@@ -162,7 +162,8 @@ test-flow-update-allowlist:
 bootstrap-typescript:
 	rm -rf ./build/typescript
 	mkdir -p ./build
-	git clone --single-branch --shallow-since=2022-04-01 https://github.com/microsoft/TypeScript.git ./build/typescript
+	git clone --filter=blob:none --sparse --shallow-since=2022-04-01 https://github.com/microsoft/TypeScript.git ./build/typescript
+	cd build/typescript && git sparse-checkout set "/tests"
 	cd build/typescript && git checkout -q $(TYPESCRIPT_COMMIT)
 
 test-typescript:

diff --git a/scripts/parser-tests/test262/index.js b/scripts/parser-tests/test262/index.js
@@ -167,7 +167,7 @@ function* getPlugins(features) {
 }
 
 const runner = new TestRunner({
-  testDir: path.join(dirname, "../../../build/test262"),
+  testDir: path.join(dirname, "../../../build/test262").replace(/\\/g, "/"),
   allowlist: path.join(dirname, "allowlist.txt"),
   logInterval: 500,
   shouldUpdate: process.argv.includes("--update-allowlist"),
@@ -179,7 +179,7 @@ const runner = new TestRunner({
 
     for await (const test of stream) {
       // strip test/
-      const fileName = test.file.slice(5);
+      const fileName = test.file.slice(5).replace(/\\/g, "/");
 
       if (ignoredTests.some(start => fileName.startsWith(start))) continue;
 

diff --git a/scripts/parser-tests/utils/parser-test-runner.js b/scripts/parser-tests/utils/parser-test-runner.js
@@ -50,6 +50,7 @@ class TestRunner {
       test.actualError = false;
     } catch (err) {
       test.actualError = true;
+      test.actualErrorObject = err;
     }
 
     test.result = test.expectedError !== test.actualError ? "fail" : "pass";
@@ -85,20 +86,61 @@ class TestRunner {
       .map(test => test.id)
       .concat(summary.unrecognized);
 
-    const updated = summary.disallowed.falsePositive
-      .concat(summary.disallowed.falseNegative)
-      .map(test => test.id);
+    const allowedFalsePositiveIds = summary.allowed.falsePositive.map(
+      test => test.id
+    );
+
+    let invalidWithoutError = [];
+    let validWithError = [];
 
     for (const line of contents.split("\n")) {
       const testId = line.replace(/#.*$/, "").trim();
-      if (!toRemove.includes(testId) && line) {
-        updated.push(line);
+      if (testId && !toRemove.includes(testId)) {
+        if (allowedFalsePositiveIds.includes(testId)) {
+          invalidWithoutError.push(line);
+        } else {
+          validWithError.push(line);
+        }
       }
     }
 
-    updated.sort();
+    invalidWithoutError = invalidWithoutError.concat(
+      summary.disallowed.falsePositive.map(test => test.id)
+    );
+    validWithError = validWithError.concat(
+      summary.disallowed.falseNegative.map(test => test.id)
+    );
+
+    invalidWithoutError.sort();
+    validWithError.sort();
+
+    const errorsMap = new Map();
+    summary.allowed.falseNegative
+      .concat(summary.disallowed.falseNegative)
+      .forEach(test => {
+        errorsMap.set(test.id, test.actualErrorObject);
+      });
+
+    const updated = [
+      `# ${invalidWithoutError.length} invalid programs did not produce a parsing error\n`,
+      "\n",
+      invalidWithoutError.join("\n"),
+      "\n",
+      "\n",
+      "\n",
+      `# ${validWithError.length} valid programs produced a parsing error\n`,
+      "\n",
+      ...validWithError
+        // .map(
+        //   v =>
+        //     `${v}\n# ${JSON.stringify(errorsMap.get(v), null, 2)
+        //       .trimEnd()
+        //       .replace(/\n/g, "\n#")}\n`
+        // )
+        .join("\n"),
+    ];
 
-    await fs.writeFile(this.allowlist, updated.join("\n") + "\n", "utf8");
+    await fs.writeFile(this.allowlist, updated.join("") + "\n", "utf8");
   }
 
   interpret(results, allowlist) {