From 296cb6340ffa8a99552348cbd8ec29c1d97e4592 Mon Sep 17 00:00:00 2001 From: Heat Hamilton Date: Tue, 10 Dec 2024 18:24:43 -0500 Subject: [PATCH 1/6] Fixed versioning plugins; wip test --- scripts/add-version-to-nav-data.mjs | 53 +++++++++---------- .../add-version-to-internal-links.mjs | 14 +++-- .../build-mdx-transforms-file.mjs | 15 ++++-- .../rewrite-internal-redirects.mjs | 4 +- scripts/prebuild.mjs | 7 ++- 5 files changed, 50 insertions(+), 43 deletions(-) diff --git a/scripts/add-version-to-nav-data.mjs b/scripts/add-version-to-nav-data.mjs index 0356a0db14..e97d2976f6 100644 --- a/scripts/add-version-to-nav-data.mjs +++ b/scripts/add-version-to-nav-data.mjs @@ -1,23 +1,6 @@ import fs from 'node:fs/promises' - -const terraformBasePaths = [ - '/cdktf', - '/cli', - '/cloud-docs/agents', - '/cloud-docs', - '/docs', - '/enterprise', - '/internals', - '/intro', - '/language', - '/plugin', - '/plugin/framework', - '/plugin/log', - '/plugin/mux', - '/plugin/sdkv2', - '/plugin/testing', - '/registry', -] +import { ALL_REPO_CONFIG } from './migrate-content/repo-config.mjs' +import semver from 'semver' /** * Adds version information to navigation data in a JSON file. @@ -39,13 +22,29 @@ export async function addVersionToNavData(filePath, versionMetadata) { } try { + const relativePath = filePath.split('content/')[1] + /** + * product and version variables, which are assigned based on the + * specific indices those strings are expected to be in the filepath + */ + const [product, version] = relativePath.split('/') + + // We are looking at a versionless doc + if (!semver.valid(semver.coerce(version))) { + return + } + + if (!versionMetadata[product]) { + throw new Error(`No version metadata found for product: ${product}`) + } + const data = await fs.readFile(filePath, 'utf-8') if (data === '') { console.error(`File is empty: ${filePath}`) return } - const jsonData = JSON.parse(data) + const versionMatch = filePath.match(/\/content\/[^/]+\/([^/]+)\/data\//) if (!versionMatch) { @@ -53,10 +52,6 @@ export async function addVersionToNavData(filePath, versionMetadata) { return } - const version = versionMatch[1] - let product = filePath.split('/content/')[1] - product = product.split('/')[0] - // Use app/api/versionMetadata.json to get the latest version const latestVersion = versionMetadata[product].find((version) => { return version.isLatest @@ -75,9 +70,13 @@ export async function addVersionToNavData(filePath, versionMetadata) { !obj[key].includes(version) ) { // href allows linking outside of content subpath - const basePath = terraformBasePaths.find((basePath) => { - return obj[key].startsWith(basePath) - }) + let basePath = ALL_REPO_CONFIG[product].basePaths?.find( + (basePath) => { + return obj[key].startsWith(`/${basePath}`) + }, + ) + basePath = + typeof basePath === 'undefined' ? undefined : `/${basePath}` // if the href starts with a basepath, e.g. "/cli", add version after the basepath if (basePath && basePath.length) { diff --git a/scripts/mdx-transforms/add-version-to-internal-links/add-version-to-internal-links.mjs b/scripts/mdx-transforms/add-version-to-internal-links/add-version-to-internal-links.mjs index e50c0cf197..0fd7cd4199 100644 --- a/scripts/mdx-transforms/add-version-to-internal-links/add-version-to-internal-links.mjs +++ b/scripts/mdx-transforms/add-version-to-internal-links/add-version-to-internal-links.mjs @@ -16,14 +16,18 @@ import { PRODUCT_CONFIG } from '../../../app/utils/productConfig.mjs' */ export const rewriteInternalLinksPlugin = ({ entry, versionMetadata }) => { const relativePath = entry.filePath.split('content/')[1] + console.log('entry.filePath: ', entry.filePath) /** * product and version variables, which are assigned based on the * specific indices those strings are expected to be in the filepath */ const [product, version] = relativePath.split('/') - + console.log({ product, version }) // We are looking at a versionless doc - if (!semver.valid(semver.coerce(version))) { + // console.log({ + // 'semver.valid(semver.coerce(version)': semver.valid(semver.coerce(version)), + // }) + if (semver.valid(semver.coerce(version)) === null) { return } @@ -33,7 +37,7 @@ export const rewriteInternalLinksPlugin = ({ entry, versionMetadata }) => { const latestVersion = versionMetadata[product].find((version) => { return version.isLatest }).version - const basePaths = PRODUCT_CONFIG[product].basePaths || [] + const basePaths = ALL_REPO_CONFIG[product]?.basePaths || [] /** * If the version in the filepath is the latest version or * no base paths exist for the product, then skip rewriting internal links @@ -50,13 +54,13 @@ export const rewriteInternalLinksPlugin = ({ entry, versionMetadata }) => { const isLinkToRewritePattern = new RegExp( `^(?!https?:\\/\\/|http:\\/\\/)(((\\.+\\/)*)|\\/|\\/${product}(?:\\/${basePaths.join('|')})?\\/)`, ) - // Creates a regex pattern to match and replace internal links based on the provided base paths. const replacePattern = new RegExp(`/(${basePaths.join('|')})(/)?`) return function transformer(tree) { // Transforms the syntax tree by rewriting internal links to include the version. return flatMap(tree, (node) => { + console.log({ node }) // Check if the node is a link and matches the pattern for links to rewrite if (node.type === 'link' && isLinkToRewritePattern.test(node.url)) { // Replace the matched part of the URL with the versioned path @@ -83,6 +87,6 @@ export const transformRewriteInternalLinks = async ( versionMetadata, }) .process(content) - + console.log({ document }) return document.contents } diff --git a/scripts/mdx-transforms/build-mdx-transforms-file.mjs b/scripts/mdx-transforms/build-mdx-transforms-file.mjs index 967d2880e0..b9e20597af 100644 --- a/scripts/mdx-transforms/build-mdx-transforms-file.mjs +++ b/scripts/mdx-transforms/build-mdx-transforms-file.mjs @@ -5,17 +5,18 @@ import path from 'path' import remark from 'remark' import remarkMdx from 'remark-mdx' import grayMatter from 'gray-matter' +import semver from 'semver' import { paragraphCustomAlertsPlugin } from './paragraph-custom-alert/paragraph-custom-alert.mjs' -// import { rewriteInternalLinksPlugin } from './add-version-to-internal-links/add-version-to-internal-links.mjs' +import { rewriteInternalLinksPlugin } from './add-version-to-internal-links/add-version-to-internal-links.mjs' import { remarkIncludePartialsPlugin } from './include-partials/remark-include-partials.mjs' import { rewriteInternalRedirectsPlugin, loadRedirects, } from './rewrite-internal-redirects/rewrite-internal-redirects.mjs' -// const CWD = process.cwd() -// const VERSION_METADATA_FILE = path.join(CWD, 'app/api/versionMetadata.json') +const CWD = process.cwd() +const VERSION_METADATA_FILE = path.join(CWD, 'app/api/versionMetadata.json') /** * Given a file path, @@ -73,7 +74,10 @@ export async function buildFileMdxTransforms(filePath) { export async function applyFileMdxTransforms(entry) { try { const { filePath, partialsDir, outPath, version, redirectsDir } = entry - const redirects = await loadRedirects(version, redirectsDir) + let redirects + if (!semver.valid(semver.coerce(version))) { + redirects = await loadRedirects(version, redirectsDir) + } const fileString = fs.readFileSync(filePath, 'utf8') @@ -86,7 +90,7 @@ export async function applyFileMdxTransforms(entry) { .use(rewriteInternalRedirectsPlugin, { redirects, }) - // .use(rewriteInternalLinksPlugin, { entry, VERSION_METADATA_FILE }) + .use(rewriteInternalLinksPlugin, { entry, VERSION_METADATA_FILE }) .process(content) const transformedContent = String(remarkResults) @@ -94,6 +98,7 @@ export async function applyFileMdxTransforms(entry) { const transformedFileString = grayMatter.stringify(transformedContent, data) // Ensure the parent directory for the output file path exists const outDir = path.dirname(outPath) + console.log({ outDir }) if (!fs.existsSync(outDir)) { fs.mkdirSync(outDir, { recursive: true }) } diff --git a/scripts/mdx-transforms/rewrite-internal-redirects/rewrite-internal-redirects.mjs b/scripts/mdx-transforms/rewrite-internal-redirects/rewrite-internal-redirects.mjs index adb8050c76..977bd1981b 100644 --- a/scripts/mdx-transforms/rewrite-internal-redirects/rewrite-internal-redirects.mjs +++ b/scripts/mdx-transforms/rewrite-internal-redirects/rewrite-internal-redirects.mjs @@ -11,7 +11,7 @@ import * as pathToRegexp from 'path-to-regexp' * Loads redirects from the file-system and "caches" them in memory. */ const cachedRedirects = {} -export const loadRedirects = async (version, redirectsDir) => { +export const loadRedirects = async (version = 'default', redirectsDir) => { // Return the cached redirects if they are already present if (cachedRedirects[version]?.length > 0) { return cachedRedirects[version] @@ -125,7 +125,7 @@ export const rewriteInternalRedirectsPlugin = ({ redirects }) => { */ export const transformRewriteInternalRedirects = async ( mdxString, - version, + version = 'default', redirectsDir, ) => { const redirects = await loadRedirects(version, redirectsDir) diff --git a/scripts/prebuild.mjs b/scripts/prebuild.mjs index 884f59e4e5..35ddc4e839 100644 --- a/scripts/prebuild.mjs +++ b/scripts/prebuild.mjs @@ -50,10 +50,9 @@ async function copyNavDataFiles(sourceDir, destDir, versionMetadata = {}) { fs.mkdirSync(parentDir, { recursive: true }) } fs.copyFileSync(filePath, destPath) - if (!Object.keys(versionMetadata).length) { - // add version to nav data paths/hrefs - await addVersionToNavData(destPath, versionMetadata) - } + + // add version to nav data paths/hrefs + await addVersionToNavData(destPath, versionMetadata) }, 16, ) From 8def5b291f6eb675c26e9e2e66426b070ec577e5 Mon Sep 17 00:00:00 2001 From: Heat Hamilton Date: Thu, 12 Dec 2024 15:55:32 -0500 Subject: [PATCH 2/6] Remove unused semver var --- scripts/add-version-to-nav-data.mjs | 1 - .../add-version-to-internal-links.mjs | 10 ++-------- .../mdx-transforms/build-mdx-transforms-file.mjs | 16 +++++++--------- .../build-mdx-transforms-file.test.mjs | 3 ++- 4 files changed, 11 insertions(+), 19 deletions(-) diff --git a/scripts/add-version-to-nav-data.mjs b/scripts/add-version-to-nav-data.mjs index e97d2976f6..9f069767e9 100644 --- a/scripts/add-version-to-nav-data.mjs +++ b/scripts/add-version-to-nav-data.mjs @@ -44,7 +44,6 @@ export async function addVersionToNavData(filePath, versionMetadata) { return } const jsonData = JSON.parse(data) - const versionMatch = filePath.match(/\/content\/[^/]+\/([^/]+)\/data\//) if (!versionMatch) { diff --git a/scripts/mdx-transforms/add-version-to-internal-links/add-version-to-internal-links.mjs b/scripts/mdx-transforms/add-version-to-internal-links/add-version-to-internal-links.mjs index 0fd7cd4199..44f4e5a89d 100644 --- a/scripts/mdx-transforms/add-version-to-internal-links/add-version-to-internal-links.mjs +++ b/scripts/mdx-transforms/add-version-to-internal-links/add-version-to-internal-links.mjs @@ -16,17 +16,13 @@ import { PRODUCT_CONFIG } from '../../../app/utils/productConfig.mjs' */ export const rewriteInternalLinksPlugin = ({ entry, versionMetadata }) => { const relativePath = entry.filePath.split('content/')[1] - console.log('entry.filePath: ', entry.filePath) /** * product and version variables, which are assigned based on the * specific indices those strings are expected to be in the filepath */ const [product, version] = relativePath.split('/') - console.log({ product, version }) + // We are looking at a versionless doc - // console.log({ - // 'semver.valid(semver.coerce(version)': semver.valid(semver.coerce(version)), - // }) if (semver.valid(semver.coerce(version)) === null) { return } @@ -37,7 +33,7 @@ export const rewriteInternalLinksPlugin = ({ entry, versionMetadata }) => { const latestVersion = versionMetadata[product].find((version) => { return version.isLatest }).version - const basePaths = ALL_REPO_CONFIG[product]?.basePaths || [] + const basePaths = ALL_REPO_CONFIG[product].basePaths || [] /** * If the version in the filepath is the latest version or * no base paths exist for the product, then skip rewriting internal links @@ -60,7 +56,6 @@ export const rewriteInternalLinksPlugin = ({ entry, versionMetadata }) => { return function transformer(tree) { // Transforms the syntax tree by rewriting internal links to include the version. return flatMap(tree, (node) => { - console.log({ node }) // Check if the node is a link and matches the pattern for links to rewrite if (node.type === 'link' && isLinkToRewritePattern.test(node.url)) { // Replace the matched part of the URL with the versioned path @@ -87,6 +82,5 @@ export const transformRewriteInternalLinks = async ( versionMetadata, }) .process(content) - console.log({ document }) return document.contents } diff --git a/scripts/mdx-transforms/build-mdx-transforms-file.mjs b/scripts/mdx-transforms/build-mdx-transforms-file.mjs index b9e20597af..2436037e0f 100644 --- a/scripts/mdx-transforms/build-mdx-transforms-file.mjs +++ b/scripts/mdx-transforms/build-mdx-transforms-file.mjs @@ -5,7 +5,6 @@ import path from 'path' import remark from 'remark' import remarkMdx from 'remark-mdx' import grayMatter from 'gray-matter' -import semver from 'semver' import { paragraphCustomAlertsPlugin } from './paragraph-custom-alert/paragraph-custom-alert.mjs' import { rewriteInternalLinksPlugin } from './add-version-to-internal-links/add-version-to-internal-links.mjs' @@ -50,7 +49,9 @@ export async function buildFileMdxTransforms(filePath) { } console.log(`🪄 Running MDX transform on ${filePath}...`) - const result = await applyFileMdxTransforms(entry) + const result = await applyFileMdxTransforms(entry, () => { + return fs.readFile(VERSION_METADATA_FILE) + }) if (result.error) { console.error(`❗ Encountered an error: ${result.error}`) } else { @@ -71,13 +72,10 @@ export async function buildFileMdxTransforms(filePath) { * @param {string} entry.outPath * @return {object} { error: string | null } */ -export async function applyFileMdxTransforms(entry) { +export async function applyFileMdxTransforms(entry, versionMetadata = {}) { try { const { filePath, partialsDir, outPath, version, redirectsDir } = entry - let redirects - if (!semver.valid(semver.coerce(version))) { - redirects = await loadRedirects(version, redirectsDir) - } + const redirects = await loadRedirects(version, redirectsDir) const fileString = fs.readFileSync(filePath, 'utf8') @@ -90,7 +88,7 @@ export async function applyFileMdxTransforms(entry) { .use(rewriteInternalRedirectsPlugin, { redirects, }) - .use(rewriteInternalLinksPlugin, { entry, VERSION_METADATA_FILE }) + .use(rewriteInternalLinksPlugin, { entry, versionMetadata }) .process(content) const transformedContent = String(remarkResults) @@ -98,7 +96,7 @@ export async function applyFileMdxTransforms(entry) { const transformedFileString = grayMatter.stringify(transformedContent, data) // Ensure the parent directory for the output file path exists const outDir = path.dirname(outPath) - console.log({ outDir }) + if (!fs.existsSync(outDir)) { fs.mkdirSync(outDir, { recursive: true }) } diff --git a/scripts/mdx-transforms/build-mdx-transforms-file.test.mjs b/scripts/mdx-transforms/build-mdx-transforms-file.test.mjs index 8bd7424f87..ae9c03f652 100644 --- a/scripts/mdx-transforms/build-mdx-transforms-file.test.mjs +++ b/scripts/mdx-transforms/build-mdx-transforms-file.test.mjs @@ -4,6 +4,7 @@ import { buildFileMdxTransforms, applyFileMdxTransforms, } from './build-mdx-transforms-file.mjs' +import versionMetadata from 'scripts/mdx-transforms/include-partials/__fixtures__/basic/versionMetadata.json' import fs from 'fs' vi.mock('fs') @@ -41,7 +42,7 @@ test('test applyFileMdxTransforms', async () => { partialsDir: '../../partials', outPath: transformedOutPath, } - const result = await applyFileMdxTransforms(entry) + const result = await applyFileMdxTransforms(entry, versionMetadata) expect(result).toStrictEqual({ error: null }) const transformedContent = fs.readFileSync(entry.outPath, 'utf8') From 84948c500a8ac8a0f2334a945927893688408020 Mon Sep 17 00:00:00 2001 From: Heat Hamilton Date: Thu, 12 Dec 2024 16:28:43 -0500 Subject: [PATCH 3/6] Change terraform-docs-common to empty array in fixture; remove version from tfc test --- __fixtures__/versionMetadata.json | 8 +------- .../add-version-to-internal-links.test.mjs | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/__fixtures__/versionMetadata.json b/__fixtures__/versionMetadata.json index bf8259821b..27ab696d95 100644 --- a/__fixtures__/versionMetadata.json +++ b/__fixtures__/versionMetadata.json @@ -382,13 +382,7 @@ "isLatest": false } ], - "terraform-docs-common": [ - { - "version": "v0.0.x", - "releaseStage": "stable", - "isLatest": true - } - ], + "terraform-docs-common": [], "terraform-plugin-framework": [ { "version": "v1.5.x", diff --git a/scripts/mdx-transforms/add-version-to-internal-links/add-version-to-internal-links.test.mjs b/scripts/mdx-transforms/add-version-to-internal-links/add-version-to-internal-links.test.mjs index 63dc4a6e1f..ed80688823 100644 --- a/scripts/mdx-transforms/add-version-to-internal-links/add-version-to-internal-links.test.mjs +++ b/scripts/mdx-transforms/add-version-to-internal-links/add-version-to-internal-links.test.mjs @@ -114,7 +114,7 @@ describe('transformRewriteInternalLinks', () => { it('should not rewrite internal links for a product with no basePaths array', async () => { const content = `[Link to cloud-docs](/cloud-docs/some-page)` const entry = { - filePath: 'content/terraform-docs-common/v0.0.x/docs/some-file.mdx', + filePath: 'content/terraform-docs-common/docs/some-file.mdx', } const expectedOutput = content + '\n' const result = await transformRewriteInternalLinks( From 283e17636261a0f81d719e108035e1c1bedcab59 Mon Sep 17 00:00:00 2001 From: Heat Hamilton Date: Thu, 12 Dec 2024 18:17:50 -0500 Subject: [PATCH 4/6] Fix nav data versions bugs --- scripts/add-version-to-nav-data.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/add-version-to-nav-data.mjs b/scripts/add-version-to-nav-data.mjs index 9f069767e9..63ecb8dd87 100644 --- a/scripts/add-version-to-nav-data.mjs +++ b/scripts/add-version-to-nav-data.mjs @@ -74,6 +74,7 @@ export async function addVersionToNavData(filePath, versionMetadata) { return obj[key].startsWith(`/${basePath}`) }, ) + basePath = typeof basePath === 'undefined' ? undefined : `/${basePath}` @@ -81,7 +82,7 @@ export async function addVersionToNavData(filePath, versionMetadata) { if (basePath && basePath.length) { obj[key] = `${basePath}/${version}${obj[key].substring(basePath.length)}` - } else { + } else if (key === 'path') { obj[key] = `${version}/${obj[key]}` } } From 559a9bc8398643b111ebfd0c5dd6e0bc5c93991b Mon Sep 17 00:00:00 2001 From: Heat Hamilton Date: Fri, 13 Dec 2024 11:19:48 -0500 Subject: [PATCH 5/6] Fix merge conflict --- .../add-version-to-internal-links.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mdx-transforms/add-version-to-internal-links/add-version-to-internal-links.mjs b/scripts/mdx-transforms/add-version-to-internal-links/add-version-to-internal-links.mjs index 44f4e5a89d..c8295d02c0 100644 --- a/scripts/mdx-transforms/add-version-to-internal-links/add-version-to-internal-links.mjs +++ b/scripts/mdx-transforms/add-version-to-internal-links/add-version-to-internal-links.mjs @@ -33,7 +33,7 @@ export const rewriteInternalLinksPlugin = ({ entry, versionMetadata }) => { const latestVersion = versionMetadata[product].find((version) => { return version.isLatest }).version - const basePaths = ALL_REPO_CONFIG[product].basePaths || [] + const basePaths = PRODUCT_CONFIG[product].basePaths || [] /** * If the version in the filepath is the latest version or * no base paths exist for the product, then skip rewriting internal links From 98fc2aa96b19953f2f0a55b825d63f59b0e98cf8 Mon Sep 17 00:00:00 2001 From: Heat Hamilton Date: Fri, 13 Dec 2024 11:32:48 -0500 Subject: [PATCH 6/6] Fix build error by importing product config instead of repo config --- scripts/add-version-to-nav-data.mjs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/add-version-to-nav-data.mjs b/scripts/add-version-to-nav-data.mjs index 63ecb8dd87..1344f8ae1f 100644 --- a/scripts/add-version-to-nav-data.mjs +++ b/scripts/add-version-to-nav-data.mjs @@ -1,5 +1,5 @@ import fs from 'node:fs/promises' -import { ALL_REPO_CONFIG } from './migrate-content/repo-config.mjs' +import { PRODUCT_CONFIG } from '../app/utils/productConfig.mjs' import semver from 'semver' /** @@ -69,11 +69,9 @@ export async function addVersionToNavData(filePath, versionMetadata) { !obj[key].includes(version) ) { // href allows linking outside of content subpath - let basePath = ALL_REPO_CONFIG[product].basePaths?.find( - (basePath) => { - return obj[key].startsWith(`/${basePath}`) - }, - ) + let basePath = PRODUCT_CONFIG[product].basePaths?.find((basePath) => { + return obj[key].startsWith(`/${basePath}`) + }) basePath = typeof basePath === 'undefined' ? undefined : `/${basePath}`