chore(test): Make it possible to ignore tests (inikulin#449)

jmbpwtw · Feb 16, 2023 · a188b4f · a188b4f
1 parent 16ca3af
commit a188b4f
Show file tree

Hide file tree

Showing 4 changed files with 91 additions and 44 deletions.
diff --git a/packages/parse5-parser-stream/test/scripting.test.ts b/packages/parse5-parser-stream/test/scripting.test.ts
@@ -14,7 +14,7 @@ generateParsingTests(
     {
         skipFragments: true,
         withoutErrors: true,
-        testSuite: [suitePath.pathname],
+        suitePath,
     },
     async (test, opts) => {
         const chunks = makeChunks(test.input);

diff --git a/packages/parse5/lib/parser/index.test.ts b/packages/parse5/lib/parser/index.test.ts
@@ -16,6 +16,43 @@ generateParsingTests('parser', 'Parser', {}, (test, opts) => ({
         : parse5.parse(test.input, opts),
 }));
 
+generateParsingTests(
+    'parser upstream',
+    'Parser',
+    {
+        withoutErrors: true,
+        suitePath: new URL('../../../../test/data/html5lib-tests/tree-construction', import.meta.url),
+        expectErrors: [
+            '271.foreign-fragment',
+            '272.foreign-fragment',
+            '309.foreign-fragment',
+            '311.foreign-fragment',
+            '318.foreign-fragment',
+            '319.foreign-fragment',
+            '329.foreign-fragment',
+            '330.foreign-fragment',
+            '331.foreign-fragment',
+            '332.foreign-fragment',
+            '333.foreign-fragment',
+            '334.foreign-fragment',
+            '335.foreign-fragment',
+            '336.foreign-fragment',
+            '337.foreign-fragment',
+            '505.search-element',
+            '506.search-element',
+            '1408.tests26',
+            '1409.tests26',
+            '1410.tests26',
+            '1411.tests26',
+        ],
+    },
+    (test, opts) => ({
+        node: test.fragmentContext
+            ? parse5.parseFragment(test.fragmentContext, test.input, opts)
+            : parse5.parse(test.input, opts),
+    })
+);
+
 describe('parser', () => {
     it('Regression - HTML5 Legacy Doctype Misparsed with htmlparser2 tree adapter (GH-45)', () => {
         const html = '<!DOCTYPE html SYSTEM "about:legacy-compat"><html><head></head><body>Hi there!</body></html>';

diff --git a/test/data/tree-construction-regression/gh40_form_in_template.dat b/test/data/tree-construction-regression/gh40_form_in_template.dat
diff --git a/test/utils/generate-parsing-tests.ts b/test/utils/generate-parsing-tests.ts
@@ -15,33 +15,31 @@ export interface TreeConstructionTestData<T extends TreeAdapterTypeMap> extends
 }
 
 export function loadTreeConstructionTestData<T extends TreeAdapterTypeMap>(
-    dataDirs: (string | URL)[],
+    dataDir: URL,
     treeAdapter: TreeAdapter<T>
 ): TreeConstructionTestData<T>[] {
     const tests: TreeConstructionTestData<T>[] = [];
 
-    for (const dataDir of dataDirs) {
-        const dataDirPath = typeof dataDir === 'string' ? dataDir : dataDir.pathname;
-        const testSetFileNames = fs.readdirSync(dataDirPath);
-        const dirName = path.basename(dataDirPath);
+    const dataDirPath = dataDir.pathname;
+    const testSetFileNames = fs.readdirSync(dataDir);
+    const dirName = path.basename(dataDirPath);
 
-        for (const fileName of testSetFileNames) {
-            if (path.extname(fileName) !== '.dat') {
-                continue;
-            }
+    for (const fileName of testSetFileNames) {
+        if (path.extname(fileName) !== '.dat') {
+            continue;
+        }
 
-            const filePath = path.join(dataDirPath, fileName);
-            const testSet = fs.readFileSync(filePath, 'utf8');
-            const setName = fileName.replace('.dat', '');
-
-            for (const test of parseDatFile(testSet, treeAdapter)) {
-                tests.push({
-                    ...test,
-                    idx: tests.length,
-                    setName,
-                    dirName,
-                });
-            }
+        const filePath = path.join(dataDirPath, fileName);
+        const testSet = fs.readFileSync(filePath, 'utf8');
+        const setName = fileName.replace('.dat', '');
+
+        for (const test of parseDatFile(testSet, treeAdapter)) {
+            tests.push({
+                ...test,
+                idx: tests.length,
+                setName,
+                dirName,
+            });
         }
     }
 
@@ -83,7 +81,7 @@ function createParsingTest<T extends TreeAdapterTypeMap>(
     test: TreeConstructionTestData<T>,
     treeAdapter: TreeAdapter<T>,
     parse: ParseMethod<T>,
-    { withoutErrors }: { withoutErrors?: boolean }
+    { withoutErrors, expectError }: { withoutErrors?: boolean; expectError?: boolean } = {}
 ): () => Promise<void> {
     return async (): Promise<void> => {
         const errs: string[] = [];
@@ -109,37 +107,62 @@ function createParsingTest<T extends TreeAdapterTypeMap>(
         const { node, chunks } = await parse(test, opts);
         const actual = serializeToDatFileFormat(node, opts.treeAdapter);
         const msg = prettyPrintParserAssertionArgs(actual, test.expected, chunks);
+        let sawError = false;
+
+        try {
+            assert.ok(actual === test.expected, msg);
+
+            if (!withoutErrors) {
+                assert.deepEqual(errs.sort(), test.expectedErrors.sort());
+            }
+        } catch (error) {
+            if (expectError) {
+                return;
+            }
+            sawError = true;
 
-        assert.ok(actual === test.expected, msg);
+            throw error;
+        }
 
-        if (!withoutErrors) {
-            assert.deepEqual(errs.sort(), test.expectedErrors.sort());
+        if (!sawError && expectError) {
+            throw new Error(`Expected error but none was thrown`);
         }
     };
 }
 
 // TODO: Stop using the fork here.
 const treePath = new URL('../data/html5lib-tests-fork/tree-construction', import.meta.url);
-const treeRegressionPath = new URL('../data/tree-construction-regression', import.meta.url);
 
 export function generateParsingTests(
     name: string,
     prefix: string,
     {
         skipFragments,
         withoutErrors,
-        testSuite = [treePath.pathname, treeRegressionPath.pathname],
-    }: { skipFragments?: boolean; withoutErrors?: boolean; testSuite?: string[] },
+        expectErrors: expectError = [],
+        suitePath = treePath,
+    }: { skipFragments?: boolean; withoutErrors?: boolean; expectErrors?: string[]; suitePath?: URL },
     parse: ParseMethod<TreeAdapterTypeMap>
 ): void {
     generateTestsForEachTreeAdapter(name, (treeAdapter) => {
-        for (const test of loadTreeConstructionTestData(testSuite, treeAdapter).filter(
+        const errorsToExpect = new Set(expectError);
+
+        for (const test of loadTreeConstructionTestData(suitePath, treeAdapter).filter(
             (test) => !skipFragments || !test.fragmentContext
         )) {
+            const expectError = errorsToExpect.delete(`${test.idx}.${test.setName}`);
+
             it(
                 `${prefix}(${test.dirName}) - ${test.idx}.${test.setName} - \`${test.input}\` (line ${test.lineNum})`,
-                createParsingTest<TreeAdapterTypeMap>(test, treeAdapter, parse, { withoutErrors })
+                createParsingTest<TreeAdapterTypeMap>(test, treeAdapter, parse, {
+                    withoutErrors,
+                    expectError,
+                })
             );
         }
+
+        if (errorsToExpect.size > 0) {
+            throw new Error(`Expected errors were not found: ${[...errorsToExpect].join(', ')}`);
+        }
     });
 }