From d787064b76414014d68532a70bff72f78df12f03 Mon Sep 17 00:00:00 2001 From: HonzaTuron Date: Tue, 9 Sep 2025 15:16:10 +0200 Subject: [PATCH 01/12] fix: Correct meta descriptions --- docusaurus.config.js | 5 +++++ package-lock.json | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index ce58992e54..76643de54c 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -370,6 +370,11 @@ module.exports = { result.frontMatter.image ??= ogImageURL.toString(); } + // Extract the first sentence (ending with . ! or ?) even if it spans multiple lines + const contentText = result.content.replace(/\n+/g, ' ').replace(/\s+/g, ' ').trim(); + const sentenceMatch = contentText.match(/^(.*?[.!?])\s/); + result.frontMatter.description = sentenceMatch ? sentenceMatch[1].trim() : contentText; + return result; }, }, diff --git a/package-lock.json b/package-lock.json index 2a62aceead..cf69dbbf5f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -75,7 +75,7 @@ }, "apify-docs-theme": { "name": "@apify/docs-theme", - "version": "1.0.208", + "version": "1.0.209", "license": "ISC", "dependencies": { "@apify/docs-search-modal": "^1.2.2", From 7145f33c6f5e277d0af28448cfd6010c95e3847f Mon Sep 17 00:00:00 2001 From: HonzaTuron Date: Tue, 9 Sep 2025 15:37:42 +0200 Subject: [PATCH 02/12] fix: non-partial front matter --- docusaurus.config.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index 76643de54c..ab98049ac6 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -368,12 +368,12 @@ module.exports = { const ogImageURL = new URL('https://apify.com/og-image/docs-article'); ogImageURL.searchParams.set('title', result.frontMatter.title); result.frontMatter.image ??= ogImageURL.toString(); - } - // Extract the first sentence (ending with . ! or ?) even if it spans multiple lines - const contentText = result.content.replace(/\n+/g, ' ').replace(/\s+/g, ' ').trim(); - const sentenceMatch = contentText.match(/^(.*?[.!?])\s/); - result.frontMatter.description = sentenceMatch ? sentenceMatch[1].trim() : contentText; + // Extract the first sentence (ending with . ! or ?) even if it spans multiple lines + const contentText = result.content.replace(/\n+/g, ' ').replace(/\s+/g, ' ').trim(); + const sentenceMatch = contentText.match(/^(.*?[.!?])\s/); + result.frontMatter.description = sentenceMatch ? sentenceMatch[1].trim() : contentText; + } return result; }, From fdd7d4f53c3557beb8199db0c0951f92696cd9a2 Mon Sep 17 00:00:00 2001 From: HonzaTuron Date: Tue, 9 Sep 2025 15:59:11 +0200 Subject: [PATCH 03/12] fix: sanitize imports and jsx --- docusaurus.config.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index ab98049ac6..4ee030af1a 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -369,10 +369,16 @@ module.exports = { ogImageURL.searchParams.set('title', result.frontMatter.title); result.frontMatter.image ??= ogImageURL.toString(); + // Remove import statements and JSX/MDX tags from content + const contentText = result.content + .replace(/import\s+[^;]+;?/g, '') // Remove import statements + .replace(/<[^>]+>/g, '') // Remove all tags (JSX/MDX) + .replace(/\n+/g, ' ') // Replace newlines with space + .replace(/\s+/g, ' ') // Collapse whitespace + .trim(); // Extract the first sentence (ending with . ! or ?) even if it spans multiple lines - const contentText = result.content.replace(/\n+/g, ' ').replace(/\s+/g, ' ').trim(); const sentenceMatch = contentText.match(/^(.*?[.!?])\s/); - result.frontMatter.description = sentenceMatch ? sentenceMatch[1].trim() : contentText; + result.frontMatter.description ??= sentenceMatch ? sentenceMatch[1].trim() : contentText; } return result; From 69fef535211968b77b757a6d452dd208b4656529 Mon Sep 17 00:00:00 2001 From: HonzaTuron Date: Tue, 9 Sep 2025 16:03:28 +0200 Subject: [PATCH 04/12] fix: remove wrong descriptions --- docusaurus.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index 4ee030af1a..821eb90e26 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -378,7 +378,7 @@ module.exports = { .trim(); // Extract the first sentence (ending with . ! or ?) even if it spans multiple lines const sentenceMatch = contentText.match(/^(.*?[.!?])\s/); - result.frontMatter.description ??= sentenceMatch ? sentenceMatch[1].trim() : contentText; + if (!result.frontMatter.description.endsWith('.')) result.frontMatter.description = sentenceMatch ? sentenceMatch[1].trim() : contentText; } return result; From 48ce8523b1c3a21c3dac9c4975b51911f4aaadba Mon Sep 17 00:00:00 2001 From: HonzaTuron Date: Tue, 9 Sep 2025 16:39:18 +0200 Subject: [PATCH 05/12] fix: test result rewrite --- docusaurus.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index 821eb90e26..1ed1c77b74 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -378,7 +378,7 @@ module.exports = { .trim(); // Extract the first sentence (ending with . ! or ?) even if it spans multiple lines const sentenceMatch = contentText.match(/^(.*?[.!?])\s/); - if (!result.frontMatter.description.endsWith('.')) result.frontMatter.description = sentenceMatch ? sentenceMatch[1].trim() : contentText; + result.frontMatter.description = sentenceMatch ? sentenceMatch[1].trim() : contentText; } return result; From 7f28bd065990d4b7a1637eb7bae30f87c33b41c2 Mon Sep 17 00:00:00 2001 From: HonzaTuron Date: Fri, 12 Sep 2025 09:42:53 +0200 Subject: [PATCH 06/12] fix: debug remark --- docusaurus.config.js | 12 +++---- package-lock.json | 75 +++++++++++++++++++++++++++++++++++++++++++- package.json | 2 ++ 3 files changed, 81 insertions(+), 8 deletions(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index 90977e7ad1..cb58ae654a 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -2,6 +2,8 @@ const { join, resolve } = require('node:path'); const clsx = require('clsx'); const { createApiPageMD, createInfoPageMD } = require('docusaurus-plugin-openapi-docs/lib/markdown'); +const remark = require('remark'); +const strip = require('strip-markdown'); const { config } = require('./apify-docs-theme'); const { collectSlugs } = require('./tools/utils/collectSlugs'); @@ -362,13 +364,9 @@ module.exports = { ogImageURL.searchParams.set('title', result.frontMatter.title); result.frontMatter.image ??= ogImageURL.toString(); - // Remove import statements and JSX/MDX tags from content - const contentText = result.content - .replace(/import\s+[^;]+;?/g, '') // Remove import statements - .replace(/<[^>]+>/g, '') // Remove all tags (JSX/MDX) - .replace(/\n+/g, ' ') // Replace newlines with space - .replace(/\s+/g, ' ') // Collapse whitespace - .trim(); + // Use remark to strip markdown and get plain text + const processed = await remark().use(strip).process(result.content); + const contentText = String(processed).replace(/\s+/g, ' ').trim(); // Extract the first sentence (ending with . ! or ?) even if it spans multiple lines const sentenceMatch = contentText.match(/^(.*?[.!?])\s/); result.frontMatter.description = sentenceMatch ? sentenceMatch[1].trim() : contentText; diff --git a/package-lock.json b/package-lock.json index daca111320..af98008a7b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -64,7 +64,9 @@ "markdownlint-cli": "^0.45.0", "patch-package": "^8.0.0", "path-browserify": "^1.0.1", + "remark": "^15.0.1", "rimraf": "^6.0.0", + "strip-markdown": "^6.0.0", "typescript": "5.9.2", "typescript-eslint": "^8.38.0" }, @@ -74,7 +76,7 @@ }, "apify-docs-theme": { "name": "@apify/docs-theme", - "version": "1.0.209", + "version": "1.0.210", "license": "ISC", "dependencies": { "@apify/docs-search-modal": "^1.2.2", @@ -26460,6 +26462,23 @@ "node": ">= 0.10" } }, + "node_modules/remark": { + "version": "15.0.1", + "resolved": "https://registry.npmjs.org/remark/-/remark-15.0.1.tgz", + "integrity": "sha512-Eht5w30ruCXgFmxVUSlNWQ9iiimq07URKeFS3hNc8cUWy1llX4KDWfyEDZRycMc+znsN9Ux5/tJ/BFdgdOwA3A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/remark-directive": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/remark-directive/-/remark-directive-3.0.1.tgz", @@ -26938,6 +26957,36 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/remark/node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/remark/node_modules/unified": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", + "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "bail": "^2.0.0", + "devlop": "^1.0.0", + "extend": "^3.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/renderkid": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/renderkid/-/renderkid-3.0.0.tgz", @@ -28526,6 +28575,30 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strip-markdown": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-markdown/-/strip-markdown-6.0.0.tgz", + "integrity": "sha512-mSa8FtUoX3ExJYDkjPUTC14xaBAn4Ik5GPQD45G5E2egAmeV3kHgVSTfIoSDggbF6Pk9stahVgqsLCNExv6jHw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/strip-markdown/node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, "node_modules/strnum": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.1.2.tgz", diff --git a/package.json b/package.json index a4aa7ff682..159b30c461 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,9 @@ "markdownlint-cli": "^0.45.0", "patch-package": "^8.0.0", "path-browserify": "^1.0.1", + "remark": "^15.0.1", "rimraf": "^6.0.0", + "strip-markdown": "^6.0.0", "typescript": "5.9.2", "typescript-eslint": "^8.38.0" }, From 54240934f77a761769318358a92e3b8530af4ea2 Mon Sep 17 00:00:00 2001 From: HonzaTuron Date: Fri, 12 Sep 2025 09:56:21 +0200 Subject: [PATCH 07/12] fix remark import --- docusaurus.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index cb58ae654a..a5e6b65648 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -2,7 +2,7 @@ const { join, resolve } = require('node:path'); const clsx = require('clsx'); const { createApiPageMD, createInfoPageMD } = require('docusaurus-plugin-openapi-docs/lib/markdown'); -const remark = require('remark'); +const { remark } = require('remark'); const strip = require('strip-markdown'); const { config } = require('./apify-docs-theme'); From a7ccdd147314a3fad73c9dadf93d896f24157661 Mon Sep 17 00:00:00 2001 From: HonzaTuron Date: Fri, 12 Sep 2025 10:04:34 +0200 Subject: [PATCH 08/12] fix strip import --- docusaurus.config.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index a5e6b65648..05e1bc754a 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -3,7 +3,6 @@ const { join, resolve } = require('node:path'); const clsx = require('clsx'); const { createApiPageMD, createInfoPageMD } = require('docusaurus-plugin-openapi-docs/lib/markdown'); const { remark } = require('remark'); -const strip = require('strip-markdown'); const { config } = require('./apify-docs-theme'); const { collectSlugs } = require('./tools/utils/collectSlugs'); @@ -364,6 +363,8 @@ module.exports = { ogImageURL.searchParams.set('title', result.frontMatter.title); result.frontMatter.image ??= ogImageURL.toString(); + const strip = (await import('strip-markdown')).default; // ESM only + // Use remark to strip markdown and get plain text const processed = await remark().use(strip).process(result.content); const contentText = String(processed).replace(/\s+/g, ' ').trim(); From def3086e7e990b53a3e7b90d744ffa4251783f77 Mon Sep 17 00:00:00 2001 From: HonzaTuron Date: Fri, 12 Sep 2025 10:27:49 +0200 Subject: [PATCH 09/12] remove imports --- docusaurus.config.js | 5 ++++- package-lock.json | 41 +++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index 05e1bc754a..d5f3724fd8 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -3,6 +3,7 @@ const { join, resolve } = require('node:path'); const clsx = require('clsx'); const { createApiPageMD, createInfoPageMD } = require('docusaurus-plugin-openapi-docs/lib/markdown'); const { remark } = require('remark'); +const mdxRemoveImports = require('remark-mdx-remove-imports'); const { config } = require('./apify-docs-theme'); const { collectSlugs } = require('./tools/utils/collectSlugs'); @@ -366,7 +367,9 @@ module.exports = { const strip = (await import('strip-markdown')).default; // ESM only // Use remark to strip markdown and get plain text - const processed = await remark().use(strip).process(result.content); + const processed = await remark() + .use(mdxRemoveImports).use(strip) + .process(result.content); const contentText = String(processed).replace(/\s+/g, ' ').trim(); // Extract the first sentence (ending with . ! or ?) even if it spans multiple lines const sentenceMatch = contentText.match(/^(.*?[.!?])\s/); diff --git a/package-lock.json b/package-lock.json index af98008a7b..7c678da3ee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -65,6 +65,7 @@ "patch-package": "^8.0.0", "path-browserify": "^1.0.1", "remark": "^15.0.1", + "remark-mdx-remove-imports": "^1.6.22", "rimraf": "^6.0.0", "strip-markdown": "^6.0.0", "typescript": "5.9.2", @@ -26773,6 +26774,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/remark-mdx/-/remark-mdx-3.1.1.tgz", "integrity": "sha512-Pjj2IYlUY3+D8x00UJsIOg5BEvfMyeI+2uLPn9VO9Wg4MEtN/VTIq2NEJQfde9PnX15KgtHyl9S0BcTnWrIuWg==", + "license": "MIT", "dependencies": { "mdast-util-mdx": "^3.0.0", "micromark-extension-mdxjs": "^3.0.0" @@ -26782,6 +26784,20 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/remark-mdx-remove-imports": { + "version": "1.6.22", + "resolved": "https://registry.npmjs.org/remark-mdx-remove-imports/-/remark-mdx-remove-imports-1.6.22.tgz", + "integrity": "sha512-lmjAXD8Ltw0TsvBzb45S+Dxx7LTJAtDaMneMAv8LAUIPEyYoKkmGbmVsiF0/pY6mhM1Q16swCmu1TN+ie/vn/A==", + "dev": true, + "license": "MIT", + "dependencies": { + "unist-util-remove": "2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/remark-parse": { "version": "11.0.0", "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", @@ -29590,6 +29606,31 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/unist-util-remove": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unist-util-remove/-/unist-util-remove-2.0.0.tgz", + "integrity": "sha512-HwwWyNHKkeg/eXRnE11IpzY8JT55JNM1YCwwU9YNCnfzk6s8GhPXrVBBZWiwLeATJbI7euvoGSzcy9M29UeW3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "unist-util-is": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-remove/node_modules/unist-util-is": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.1.0.tgz", + "integrity": "sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==", + "dev": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/unist-util-stringify-position": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", diff --git a/package.json b/package.json index 159b30c461..963c8732c7 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "patch-package": "^8.0.0", "path-browserify": "^1.0.1", "remark": "^15.0.1", + "remark-mdx-remove-imports": "^1.6.22", "rimraf": "^6.0.0", "strip-markdown": "^6.0.0", "typescript": "5.9.2", From c7a2af23ea650a00c446f921b2b66f6c043633e9 Mon Sep 17 00:00:00 2001 From: HonzaTuron Date: Fri, 12 Sep 2025 11:01:15 +0200 Subject: [PATCH 10/12] mdx plugin --- docusaurus.config.js | 3 ++- package-lock.json | 1 + package.json | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index d5f3724fd8..e423e344c1 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -365,10 +365,11 @@ module.exports = { result.frontMatter.image ??= ogImageURL.toString(); const strip = (await import('strip-markdown')).default; // ESM only + const mdx = (await import('remark-mdx')).default; // ESM only // Use remark to strip markdown and get plain text const processed = await remark() - .use(mdxRemoveImports).use(strip) + .use(mdx).use(mdxRemoveImports).use(strip) .process(result.content); const contentText = String(processed).replace(/\s+/g, ' ').trim(); // Extract the first sentence (ending with . ! or ?) even if it spans multiple lines diff --git a/package-lock.json b/package-lock.json index 7c678da3ee..b170b8784a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -65,6 +65,7 @@ "patch-package": "^8.0.0", "path-browserify": "^1.0.1", "remark": "^15.0.1", + "remark-mdx": "^3.1.1", "remark-mdx-remove-imports": "^1.6.22", "rimraf": "^6.0.0", "strip-markdown": "^6.0.0", diff --git a/package.json b/package.json index 963c8732c7..bb5c23a016 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "patch-package": "^8.0.0", "path-browserify": "^1.0.1", "remark": "^15.0.1", + "remark-mdx": "^3.1.1", "remark-mdx-remove-imports": "^1.6.22", "rimraf": "^6.0.0", "strip-markdown": "^6.0.0", From 462e9f3649f2c3aa905b80b69f9d440d98ba28ca Mon Sep 17 00:00:00 2001 From: HonzaTuron Date: Fri, 12 Sep 2025 15:41:28 +0200 Subject: [PATCH 11/12] test html removal --- docusaurus.config.js | 8 ++++++-- package-lock.json | 8 ++++++++ package.json | 1 + 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index e423e344c1..91f2524d3a 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -4,6 +4,7 @@ const clsx = require('clsx'); const { createApiPageMD, createInfoPageMD } = require('docusaurus-plugin-openapi-docs/lib/markdown'); const { remark } = require('remark'); const mdxRemoveImports = require('remark-mdx-remove-imports'); +const stripHtml = require('remark-strip-html'); const { config } = require('./apify-docs-theme'); const { collectSlugs } = require('./tools/utils/collectSlugs'); @@ -365,11 +366,14 @@ module.exports = { result.frontMatter.image ??= ogImageURL.toString(); const strip = (await import('strip-markdown')).default; // ESM only - const mdx = (await import('remark-mdx')).default; // ESM only + // const mdx = (await import('remark-mdx')).default; // ESM only // Use remark to strip markdown and get plain text const processed = await remark() - .use(mdx).use(mdxRemoveImports).use(strip) + .use(stripHtml) + // .use(mdx) + .use(mdxRemoveImports) + .use(strip) .process(result.content); const contentText = String(processed).replace(/\s+/g, ' ').trim(); // Extract the first sentence (ending with . ! or ?) even if it spans multiple lines diff --git a/package-lock.json b/package-lock.json index b170b8784a..b011b67f23 100644 --- a/package-lock.json +++ b/package-lock.json @@ -67,6 +67,7 @@ "remark": "^15.0.1", "remark-mdx": "^3.1.1", "remark-mdx-remove-imports": "^1.6.22", + "remark-strip-html": "^1.0.2", "rimraf": "^6.0.0", "strip-markdown": "^6.0.0", "typescript": "5.9.2", @@ -26960,6 +26961,13 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/remark-strip-html": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/remark-strip-html/-/remark-strip-html-1.0.2.tgz", + "integrity": "sha512-lzOlh4gp/sd9qYVUO0QNNQbUIDYM35XhBZnvp4S4bkM3OCfrvd2GQlQVPAuneeI/K5jbxTfCGDvb9UT9IEPIwA==", + "dev": true, + "license": "MIT" + }, "node_modules/remark-toc": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/remark-toc/-/remark-toc-8.0.1.tgz", diff --git a/package.json b/package.json index bb5c23a016..f28900f0e8 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "remark": "^15.0.1", "remark-mdx": "^3.1.1", "remark-mdx-remove-imports": "^1.6.22", + "remark-strip-html": "^1.0.2", "rimraf": "^6.0.0", "strip-markdown": "^6.0.0", "typescript": "5.9.2", From 77e2449fb15e8665880315272eb29f05ecafc947 Mon Sep 17 00:00:00 2001 From: HonzaTuron Date: Fri, 12 Sep 2025 15:54:50 +0200 Subject: [PATCH 12/12] revert remark experiments --- docusaurus.config.js | 22 +++----- package-lock.json | 122 ------------------------------------------- package.json | 5 -- 3 files changed, 8 insertions(+), 141 deletions(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index 91f2524d3a..3629fec129 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -2,9 +2,6 @@ const { join, resolve } = require('node:path'); const clsx = require('clsx'); const { createApiPageMD, createInfoPageMD } = require('docusaurus-plugin-openapi-docs/lib/markdown'); -const { remark } = require('remark'); -const mdxRemoveImports = require('remark-mdx-remove-imports'); -const stripHtml = require('remark-strip-html'); const { config } = require('./apify-docs-theme'); const { collectSlugs } = require('./tools/utils/collectSlugs'); @@ -365,19 +362,16 @@ module.exports = { ogImageURL.searchParams.set('title', result.frontMatter.title); result.frontMatter.image ??= ogImageURL.toString(); - const strip = (await import('strip-markdown')).default; // ESM only - // const mdx = (await import('remark-mdx')).default; // ESM only + // Remove import statements and JSX/MDX tags from content + const contentText = result.content + .replace(/import\s+[^;]+;?/g, '') // Remove import statements + .replace(/<[^>]+>/g, '') // Remove all tags (JSX/MDX) + .replace(/\n+/g, ' ') // Replace newlines with space + .replace(/\s+/g, ' ') // Collapse whitespace + .trim(); - // Use remark to strip markdown and get plain text - const processed = await remark() - .use(stripHtml) - // .use(mdx) - .use(mdxRemoveImports) - .use(strip) - .process(result.content); - const contentText = String(processed).replace(/\s+/g, ' ').trim(); - // Extract the first sentence (ending with . ! or ?) even if it spans multiple lines const sentenceMatch = contentText.match(/^(.*?[.!?])\s/); + result.frontMatter.description = sentenceMatch ? sentenceMatch[1].trim() : contentText; } diff --git a/package-lock.json b/package-lock.json index b011b67f23..9a3b8be691 100644 --- a/package-lock.json +++ b/package-lock.json @@ -64,12 +64,7 @@ "markdownlint-cli": "^0.45.0", "patch-package": "^8.0.0", "path-browserify": "^1.0.1", - "remark": "^15.0.1", - "remark-mdx": "^3.1.1", - "remark-mdx-remove-imports": "^1.6.22", - "remark-strip-html": "^1.0.2", "rimraf": "^6.0.0", - "strip-markdown": "^6.0.0", "typescript": "5.9.2", "typescript-eslint": "^8.38.0" }, @@ -26465,23 +26460,6 @@ "node": ">= 0.10" } }, - "node_modules/remark": { - "version": "15.0.1", - "resolved": "https://registry.npmjs.org/remark/-/remark-15.0.1.tgz", - "integrity": "sha512-Eht5w30ruCXgFmxVUSlNWQ9iiimq07URKeFS3hNc8cUWy1llX4KDWfyEDZRycMc+znsN9Ux5/tJ/BFdgdOwA3A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "remark-parse": "^11.0.0", - "remark-stringify": "^11.0.0", - "unified": "^11.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, "node_modules/remark-directive": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/remark-directive/-/remark-directive-3.0.1.tgz", @@ -26786,20 +26764,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/remark-mdx-remove-imports": { - "version": "1.6.22", - "resolved": "https://registry.npmjs.org/remark-mdx-remove-imports/-/remark-mdx-remove-imports-1.6.22.tgz", - "integrity": "sha512-lmjAXD8Ltw0TsvBzb45S+Dxx7LTJAtDaMneMAv8LAUIPEyYoKkmGbmVsiF0/pY6mhM1Q16swCmu1TN+ie/vn/A==", - "dev": true, - "license": "MIT", - "dependencies": { - "unist-util-remove": "2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, "node_modules/remark-parse": { "version": "11.0.0", "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", @@ -26961,13 +26925,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/remark-strip-html": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/remark-strip-html/-/remark-strip-html-1.0.2.tgz", - "integrity": "sha512-lzOlh4gp/sd9qYVUO0QNNQbUIDYM35XhBZnvp4S4bkM3OCfrvd2GQlQVPAuneeI/K5jbxTfCGDvb9UT9IEPIwA==", - "dev": true, - "license": "MIT" - }, "node_modules/remark-toc": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/remark-toc/-/remark-toc-8.0.1.tgz", @@ -26982,36 +26939,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/remark/node_modules/@types/mdast": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", - "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/unist": "*" - } - }, - "node_modules/remark/node_modules/unified": { - "version": "11.0.5", - "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", - "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0", - "bail": "^2.0.0", - "devlop": "^1.0.0", - "extend": "^3.0.0", - "is-plain-obj": "^4.0.0", - "trough": "^2.0.0", - "vfile": "^6.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, "node_modules/renderkid": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/renderkid/-/renderkid-3.0.0.tgz", @@ -28600,30 +28527,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/strip-markdown": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/strip-markdown/-/strip-markdown-6.0.0.tgz", - "integrity": "sha512-mSa8FtUoX3ExJYDkjPUTC14xaBAn4Ik5GPQD45G5E2egAmeV3kHgVSTfIoSDggbF6Pk9stahVgqsLCNExv6jHw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/strip-markdown/node_modules/@types/mdast": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", - "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/unist": "*" - } - }, "node_modules/strnum": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.1.2.tgz", @@ -29615,31 +29518,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/unist-util-remove": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/unist-util-remove/-/unist-util-remove-2.0.0.tgz", - "integrity": "sha512-HwwWyNHKkeg/eXRnE11IpzY8JT55JNM1YCwwU9YNCnfzk6s8GhPXrVBBZWiwLeATJbI7euvoGSzcy9M29UeW3g==", - "dev": true, - "license": "MIT", - "dependencies": { - "unist-util-is": "^4.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/unist-util-remove/node_modules/unist-util-is": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.1.0.tgz", - "integrity": "sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==", - "dev": true, - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, "node_modules/unist-util-stringify-position": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", diff --git a/package.json b/package.json index f28900f0e8..a4aa7ff682 100644 --- a/package.json +++ b/package.json @@ -62,12 +62,7 @@ "markdownlint-cli": "^0.45.0", "patch-package": "^8.0.0", "path-browserify": "^1.0.1", - "remark": "^15.0.1", - "remark-mdx": "^3.1.1", - "remark-mdx-remove-imports": "^1.6.22", - "remark-strip-html": "^1.0.2", "rimraf": "^6.0.0", - "strip-markdown": "^6.0.0", "typescript": "5.9.2", "typescript-eslint": "^8.38.0" },