chore: check rule examples for syntax errors (#17718)

* check rule examples for syntax errors

* fix a test on Windows

* more than three backticks allowed

* add comments, minimal tweaks

* fix Makefile task

* fix for multiple trailing spaces after 'correct'

* rename npm script

.github/workflows/ci.yml

@@ -33,6 +33,8 @@ jobs:
       run: npm run lint:scss
     - name: Lint Docs JS Files
       run: node Makefile lintDocsJS
+    - name: Check Rule Examples
+      run: node Makefile checkRuleExamples
     - name: Build Docs Website
       working-directory: docs
       run: npm run build

Makefile.js

@@ -867,6 +867,17 @@ target.checkRuleFiles = function() {
 
 };
 
+target.checkRuleExamples = function() {
+    const { execFileSync } = require("child_process");
+
+    // We don't need the stack trace of execFileSync if the command fails.
+    try {
+        execFileSync(process.execPath, ["tools/check-rule-examples.js", "docs/src/rules/*.md"], { stdio: "inherit" });
+    } catch {
+        exit(1);
+    }
+};
+
 target.checkLicenses = function() {
 
     /**

docs/.eleventy.js

@@ -14,6 +14,8 @@ const { highlighter, lineNumberPlugin } = require("./src/_plugins/md-syntax-high
 const {
     DateTime
 } = require("luxon");
+const markdownIt = require("markdown-it");
+const markdownItRuleExample = require("./tools/markdown-it-rule-example");
 
 module.exports = function(eleventyConfig) {
 
@@ -113,7 +115,7 @@ module.exports = function(eleventyConfig) {
      * Source: https://github.com/11ty/eleventy/issues/658
      */
     eleventyConfig.addFilter("markdown", value => {
-        const markdown = require("markdown-it")({
+        const markdown = markdownIt({
             html: true
         });
 
@@ -191,57 +193,39 @@ module.exports = function(eleventyConfig) {
         return btoa(unescape(encodeURIComponent(text)));
     }
 
-    /**
-     * Creates markdownItContainer settings for a playground-linked codeblock.
-     * @param {string} name Plugin name and class name to add to the code block.
-     * @returns {[string, object]} Plugin name and options for markdown-it.
-     */
-    function withPlaygroundRender(name) {
-        return [
-            name,
-            {
-                render(tokens, index) {
-                    if (tokens[index].nesting !== 1) {
-                        return "</div>";
-                    }
-
-                    // See https://github.com/eslint/eslint.org/blob/ac38ab41f99b89a8798d374f74e2cce01171be8b/src/playground/App.js#L44
-                    const parserOptionsJSON = tokens[index].info?.split("correct ")[1]?.trim();
-                    const parserOptions = { sourceType: "module", ...(parserOptionsJSON && JSON.parse(parserOptionsJSON)) };
-
-                    // Remove trailing newline and presentational `⏎` characters (https://github.com/eslint/eslint/issues/17627):
-                    const content = tokens[index + 1].content
-                        .replace(/\n$/u, "")
-                        .replace(/⏎(?=\n)/gu, "");
-                    const state = encodeToBase64(
-                        JSON.stringify({
-                            options: { parserOptions },
-                            text: content
-                        })
-                    );
-                    const prefix = process.env.CONTEXT && process.env.CONTEXT !== "deploy-preview"
-                        ? ""
-                        : "https://eslint.org";
-
-                    return `
-                        <div class="${name}">
+    // markdown-it plugin options for playground-linked code blocks in rule examples.
+    const ruleExampleOptions = markdownItRuleExample({
+        open(type, code, parserOptions) {
+
+            // See https://github.com/eslint/eslint.org/blob/ac38ab41f99b89a8798d374f74e2cce01171be8b/src/playground/App.js#L44
+            const state = encodeToBase64(
+                JSON.stringify({
+                    options: { parserOptions },
+                    text: code
+                })
+            );
+            const prefix = process.env.CONTEXT && process.env.CONTEXT !== "deploy-preview"
+                ? ""
+                : "https://eslint.org";
+
+            return `
+                        <div class="${type}">
                             <a class="c-btn c-btn--secondary c-btn--playground" href="${prefix}/play#${state}" target="_blank">
                                 Open in Playground
                             </a>
-                    `.trim();
-                }
-            }
-        ];
-    }
+            `.trim();
+        },
+        close() {
+            return "</div>";
+        }
+    });
 
-    const markdownIt = require("markdown-it");
     const md = markdownIt({ html: true, linkify: true, typographer: true, highlight: (str, lang) => highlighter(md, str, lang) })
         .use(markdownItAnchor, {
             slugify: s => slug(s)
         })
         .use(markdownItContainer, "img-container", {})
-        .use(markdownItContainer, ...withPlaygroundRender("correct"))
-        .use(markdownItContainer, ...withPlaygroundRender("incorrect"))
+        .use(markdownItContainer, "rule-example", ruleExampleOptions)
         .use(markdownItContainer, "warning", {
             render(tokens, idx) {
                 return generateAlertMarkup("warning", tokens, idx);

docs/tools/markdown-it-rule-example.js

@@ -0,0 +1,88 @@
+"use strict";
+
+/** @typedef {import("../../lib/shared/types").ParserOptions} ParserOptions */
+
+/**
+ * A callback function to handle the opening of container blocks.
+ * @callback OpenHandler
+ * @param {"correct" | "incorrect"} type The type of the example.
+ * @param {string} code The example code.
+ * @param {ParserOptions} parserOptions The parser options to be passed to the Playground.
+ * @param {Object} codeBlockToken The `markdown-it` token for the code block inside the container.
+ * @returns {string | undefined} If a text is returned, it will be appended to the rendered output
+ * of `markdown-it`.
+ */
+
+/**
+ * A callback function to handle the closing of container blocks.
+ * @callback CloseHandler
+ * @returns {string | undefined} If a text is returned, it will be appended to the rendered output
+ * of `markdown-it`.
+ */
+
+/**
+ * This is a utility to simplify the creation of `markdown-it-container` options to handle rule
+ * examples in the documentation.
+ * It is designed to automate the following common tasks:
+ *
+ * - Ensure that the plugin instance only matches container blocks tagged with 'correct' or
+ * 'incorrect'.
+ * - Parse the optional `parserOptions` after the correct/incorrect tag.
+ * - Apply common transformations to the code inside the code block, like stripping '⏎' at the end
+ * of a line or the last newline character.
+ *
+ * Additionally, the opening and closing of the container blocks are handled by two distinct
+ * callbacks, of which only the `open` callback is required.
+ * @param {Object} options The options object.
+ * @param {OpenHandler} options.open The open callback.
+ * @param {CloseHandler} [options.close] The close callback.
+ * @returns {Object} The `markdown-it-container` options.
+ * @example
+ * const markdownIt = require("markdown-it");
+ * const markdownItContainer = require("markdown-it-container");
+ *
+ * markdownIt()
+ *     .use(markdownItContainer, "rule-example", markdownItRuleExample({
+ *         open(type, code, parserOptions, codeBlockToken) {
+ *             // do something
+ *         }
+ *         close() {
+ *             // do something
+ *         }
+ *     }))
+ *     .render(text);
+ *
+ */
+function markdownItRuleExample({ open, close }) {
+    return {
+        validate(info) {
+            return /^\s*(?:in)?correct(?!\S)/u.test(info);
+        },
+        render(tokens, index) {
+            const tagToken = tokens[index];
+
+            if (tagToken.nesting < 0) {
+                const text = close ? close() : void 0;
+
+                // Return an empty string to avoid appending unexpected text to the output.
+                return typeof text === "string" ? text : "";
+            }
+
+            const { type, parserOptionsJSON } = /^\s*(?<type>\S+)(\s+(?<parserOptionsJSON>\S.*?))?\s*$/u.exec(tagToken.info).groups;
+            const parserOptions = { sourceType: "module", ...(parserOptionsJSON && JSON.parse(parserOptionsJSON)) };
+            const codeBlockToken = tokens[index + 1];
+
+            // Remove trailing newline and presentational `⏎` characters (https://github.com/eslint/eslint/issues/17627):
+            const code = codeBlockToken.content
+                .replace(/\n$/u, "")
+                .replace(/⏎(?=\n)/gu, "");
+
+            const text = open(type, code, parserOptions, codeBlockToken);
+
+            // Return an empty string to avoid appending unexpected text to the output.
+            return typeof text === "string" ? text : "";
+        }
+    };
+}
+
+module.exports = markdownItRuleExample;

package.json

@@ -19,6 +19,7 @@
     "build:readme": "node tools/update-readme.js",
     "lint": "node Makefile.js lint",
     "lint:docs:js": "node Makefile.js lintDocsJS",
+    "lint:docs:rule-examples": "node Makefile.js checkRuleExamples",
     "lint:fix": "node Makefile.js lint -- fix",
     "lint:fix:docs:js": "node Makefile.js lintDocsJS -- fix",
     "release:generate:alpha": "node Makefile.js generatePrerelease -- alpha",
@@ -42,6 +43,7 @@
       "git add packages/js/src/configs/eslint-all.js"
     ],
     "docs/src/rules/*.md": [
+      "node tools/check-rule-examples.js",
       "node tools/fetch-docs-links.js",
       "git add docs/src/_data/further_reading_links.json"
     ],
@@ -132,6 +134,8 @@
     "gray-matter": "^4.0.3",
     "lint-staged": "^11.0.0",
     "load-perf": "^0.2.0",
+    "markdown-it": "^12.2.0",
+    "markdown-it-container": "^3.0.0",
     "markdownlint": "^0.31.1",
     "markdownlint-cli": "^0.37.0",
     "marked": "^4.0.8",

tests/fixtures/bad-examples.md

@@ -0,0 +1,26 @@
+---
+title: Lorem Ipsum
+---
+
+This file contains rule example code with syntax errors.
+
+<!-- markdownlint-capture -->
+<!-- markdownlint-disable MD040 -->
+::: incorrect { "sourceType": "script" }
+
+```
+export default "foo";
+```
+
+:::
+<!-- markdownlint-restore -->
+
+:::correct
+
+````ts
+const foo = "bar";
+
+const foo = "baz";
+````
+
+:::

tests/fixtures/good-examples.md

@@ -0,0 +1,33 @@
+This file contains rule example code without syntax errors.
+
+::: incorrect
+
+```js
+export default⏎
+"foo";
+```
+
+:::
+
+::: correct { "ecmaFeatures": { "jsx": true } }
+
+```jsx
+const foo = <bar></bar>;
+```
+
+:::
+
+A test with multiple spaces after 'correct':
+<!-- markdownlint-disable-next-line no-trailing-spaces -->
+:::correct  
+
+```js
+```
+
+:::
+
+The following code block is not a rule example, so it won't be checked:
+
+```js
+!@#$%^&*()
+```

tests/tools/check-rule-examples.js

@@ -0,0 +1,95 @@
+"use strict";
+
+//------------------------------------------------------------------------------
+// Requirements
+//------------------------------------------------------------------------------
+
+const assert = require("assert");
+const { execFile } = require("child_process");
+const { promisify } = require("util");
+
+//------------------------------------------------------------------------------
+// Helpers
+//------------------------------------------------------------------------------
+
+/**
+ * Runs check-rule-examples on the specified files.
+ * @param {...string} filenames Files to be passed to check-rule-examples.
+ * @returns {Promise<ChildProcess>} An object with properties `stdout` and `stderr` on success.
+ * @throws An object with properties `code`, `stdout` and `stderr` on success.
+ */
+async function runCheckRuleExamples(...filenames) {
+    return await promisify(execFile)(
+        process.execPath,
+        ["--no-deprecation", "tools/check-rule-examples.js", ...filenames],
+        { env: { FORCE_COLOR: "3" } } // 24-bit color mode
+    );
+}
+
+//------------------------------------------------------------------------------
+// Tests
+//------------------------------------------------------------------------------
+
+describe("check-rule-examples", () => {
+
+    it("succeeds when not passed any files", async () => {
+        const childProcess = await runCheckRuleExamples();
+
+        assert.strictEqual(childProcess.stdout, "");
+        assert.strictEqual(childProcess.stderr, "");
+    });
+
+    it("succeeds when passed a syntax error free file", async () => {
+        const childProcess = await runCheckRuleExamples("tests/fixtures/good-examples.md");
+
+        assert.strictEqual(childProcess.stdout, "");
+        assert.strictEqual(childProcess.stderr, "");
+    });
+
+    it("fails when passed a file with a syntax error", async () => {
+        const promise = runCheckRuleExamples("tests/fixtures/good-examples.md", "tests/fixtures/bad-examples.md");
+
+        await assert.rejects(
+            promise,
+            {
+                code: 1,
+                stdout: "",
+                stderr:
+                "\x1B[0m\x1B[0m\n" +
+                "\x1B[0m\x1B[4mtests/fixtures/bad-examples.md\x1B[24m\x1B[0m\n" +
+                "\x1B[0m  \x1B[2m11:4\x1B[22m  \x1B[31merror\x1B[39m  Missing language tag: use one of 'javascript', 'js' or 'jsx'\x1B[0m\n" +
+                "\x1B[0m  \x1B[2m12:1\x1B[22m  \x1B[31merror\x1B[39m  Syntax error: 'import' and 'export' may appear only with 'sourceType: module'\x1B[0m\n" +
+                "\x1B[0m  \x1B[2m20:5\x1B[22m  \x1B[31merror\x1B[39m  Nonstandard language tag 'ts': use one of 'javascript', 'js' or 'jsx'\x1B[0m\n" +
+                "\x1B[0m  \x1B[2m23:7\x1B[22m  \x1B[31merror\x1B[39m  Syntax error: Identifier 'foo' has already been declared\x1B[0m\n" +
+                "\x1B[0m\x1B[0m\n" +
+                "\x1B[0m\x1B[31m\x1B[1m✖ 4 problems (4 errors, 0 warnings)\x1B[22m\x1B[39m\x1B[0m\n" +
+                "\x1B[0m\x1B[31m\x1B[1m\x1B[22m\x1B[39m\x1B[0m\n"
+            }
+        );
+    });
+
+    it("fails when a file cannot be processed", async () => {
+        const promise = runCheckRuleExamples("tests/fixtures/non-existing-examples.md");
+
+        await assert.rejects(
+            promise,
+            ({ code, stdout, stderr }) => {
+                assert.strictEqual(code, 1);
+                assert.strictEqual(stdout, "");
+                const expectedStderr =
+                "\x1B[0m\x1B[0m\n" +
+                "\x1B[0m\x1B[4mtests/fixtures/non-existing-examples.md\x1B[24m\x1B[0m\n" +
+                "\x1B[0m  \x1B[2m0:0\x1B[22m  \x1B[31merror\x1B[39m  Error checking file: ENOENT: no such file or directory, open <FILE>\x1B[0m\n" +
+                "\x1B[0m\x1B[0m\n" +
+                "\x1B[0m\x1B[31m\x1B[1m✖ 1 problem (1 error, 0 warnings)\x1B[22m\x1B[39m\x1B[0m\n" +
+                "\x1B[0m\x1B[31m\x1B[1m\x1B[22m\x1B[39m\x1B[0m\n";
+
+                // Replace filename as it's OS-dependent.
+                const normalizedStderr = stderr.replace(/'.+'/u, "<FILE>");
+
+                assert.strictEqual(normalizedStderr, expectedStderr);
+                return true;
+            }
+        );
+    });
+});