diff --git a/.gitignore b/.gitignore index 96cee29b2e..52cfe5d9bb 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,7 @@ tfe-releases-repos.json scripts/prebuild/prebuild-arm-mac-binary scripts/prebuild/prebuild-x64-linux-binary scripts/prebuild/prebuild-arm-linux-binary + +scripts/prebuild/mdx-transforms/benchmarks + +scripts/prebuild/mdx-transforms/content-exclusion-OOP-approach \ No newline at end of file diff --git a/content/vault/v1.20.x/content/docs/concepts/client-count/test-vault-exclusion.mdx b/content/vault/v1.20.x/content/docs/concepts/client-count/test-vault-exclusion.mdx new file mode 100644 index 0000000000..bfae94aa3d --- /dev/null +++ b/content/vault/v1.20.x/content/docs/concepts/client-count/test-vault-exclusion.mdx @@ -0,0 +1,25 @@ +--- + page_title: Test Vault Content Exclusion yoo +--- + +# Test Vault Content Exclusion + +This content should always appear. + + +This content should be REMOVED because current version (1.20.x) is less than 1.21.x + + + +This content should STAY because current version (1.20.x) is less than or equal to 1.21.x + + + +This content should STAY because current version equals 1.20.x + + + +This content shofuld be REMOVED because current version (1.20.x) is not less than 1.19.x + + +Final content that should always appear. \ No newline at end of file diff --git a/productConfig.mjs b/productConfig.mjs index 3d1717b6ef..2e0aea3f8d 100644 --- a/productConfig.mjs +++ b/productConfig.mjs @@ -175,6 +175,7 @@ export const PRODUCT_CONFIG = { contentDir: 'docs', dataDir: 'data', productSlug: 'terraform', + supportsExclusionDirectives: true, /** * Note: we need to sort versions for various reasons. Nearly all * our documentation is semver-versioned. PTFE is not. Rather than @@ -310,6 +311,7 @@ export const PRODUCT_CONFIG = { semverCoerce: semver.coerce, versionedDocs: false, websiteDir: 'website', + supportsExclusionDirectives: true, }, 'terraform-plugin-framework': { /** @@ -450,6 +452,7 @@ export const PRODUCT_CONFIG = { semverCoerce: semver.coerce, versionedDocs: true, websiteDir: 'website', + supportsExclusionDirectives: true, }, 'well-architected-framework': { /** diff --git a/scripts/prebuild/mdx-transforms/build-mdx-transforms-file.mjs b/scripts/prebuild/mdx-transforms/build-mdx-transforms-file.mjs index d05326d3b4..c851f363b7 100644 --- a/scripts/prebuild/mdx-transforms/build-mdx-transforms-file.mjs +++ b/scripts/prebuild/mdx-transforms/build-mdx-transforms-file.mjs @@ -18,7 +18,8 @@ import { rewriteInternalRedirectsPlugin, loadRedirects, } from './rewrite-internal-redirects/rewrite-internal-redirects.mjs' -import { transformExcludeTerraformContent } from './exclude-terraform-content/index.mjs' +import { transformExcludeContent } from './exclude-content/index.mjs' +import { PRODUCT_CONFIG } from '#productConfig.mjs' const CWD = process.cwd() const VERSION_METADATA_FILE = path.join(CWD, 'app/api/versionMetadata.json') @@ -86,10 +87,28 @@ export async function applyFileMdxTransforms(entry, versionMetadata = {}) { const { data, content } = grayMatter(fileString) - const remarkResults = await remark() + // Check if this file is in a global/partials directory + // Global partials should not have content exclusion applied to them + // as they are version-agnostic and shared across all versions + const isGlobalPartial = filePath.includes('/global/partials/') + + const processor = remark() .use(remarkMdx) - .use(transformExcludeTerraformContent, { filePath }) + // Process partials first, then content exclusion + // This ensures exclusion directives in global partials are properly evaluated .use(remarkIncludePartialsPlugin, { partialsDir, filePath }) + + // Only apply content exclusion if this is NOT a global partial + if (!isGlobalPartial) { + processor.use(transformExcludeContent, { + filePath, + version, + repoSlug: entry.repoSlug, + productConfig: PRODUCT_CONFIG[entry.repoSlug], + }) + } + + const remarkResults = await processor .use(paragraphCustomAlertsPlugin) .use(rewriteInternalRedirectsPlugin, { redirects, diff --git a/scripts/prebuild/mdx-transforms/build-mdx-transforms.mjs b/scripts/prebuild/mdx-transforms/build-mdx-transforms.mjs index 4f93fcd86c..3b75564824 100644 --- a/scripts/prebuild/mdx-transforms/build-mdx-transforms.mjs +++ b/scripts/prebuild/mdx-transforms/build-mdx-transforms.mjs @@ -23,7 +23,8 @@ import { rewriteInternalRedirectsPlugin, loadRedirects, } from './rewrite-internal-redirects/rewrite-internal-redirects.mjs' -import { transformExcludeTerraformContent } from './exclude-terraform-content/index.mjs' + +import { transformExcludeContent } from './exclude-content/index.mjs' import { PRODUCT_CONFIG } from '#productConfig.mjs' @@ -78,11 +79,9 @@ export async function buildMdxTransforms( ) const redirectsDir = path.join(targetDir, repoSlug, verifiedVersion) const outPath = path.join(outputDir, relativePath) - return { filePath, partialsDir, outPath, version, redirectsDir } + return { repoSlug, filePath, partialsDir, outPath, version, redirectsDir } }) - /** - * Apply MDX transforms to each file entry, in batches - */ + console.log(`Running MDX transforms on ${mdxFileEntries.length} files...`) const results = await batchPromises( 'MDX transforms', @@ -132,10 +131,28 @@ async function applyMdxTransforms(entry, versionMetadata = {}) { const fileString = fs.readFileSync(filePath, 'utf8') const { data, content } = grayMatter(fileString) - const remarkResults = await remark() + // Check if this file is in a global/partials directory + // Global partials should not have content exclusion applied to them + // as they are version-agnostic and shared across all versions + const isGlobalPartial = filePath.includes('/global/partials/') + + const processor = remark() .use(remarkMdx) - .use(transformExcludeTerraformContent, { filePath }) + // Process partials first, then content exclusion + // This ensures exclusion directives in global partials are properly evaluated .use(remarkIncludePartialsPlugin, { partialsDir, filePath }) + + // Only apply content exclusion if this is NOT a global partial + if (!isGlobalPartial) { + processor.use(transformExcludeContent, { + filePath, + version, + repoSlug: entry.repoSlug, + productConfig: PRODUCT_CONFIG[entry.repoSlug], + }) + } + + const remarkResults = await processor .use(paragraphCustomAlertsPlugin) .use(rewriteInternalRedirectsPlugin, { redirects, diff --git a/scripts/prebuild/mdx-transforms/build-mdx-transforms.test.mjs b/scripts/prebuild/mdx-transforms/build-mdx-transforms.test.mjs new file mode 100644 index 0000000000..cd228de35f --- /dev/null +++ b/scripts/prebuild/mdx-transforms/build-mdx-transforms.test.mjs @@ -0,0 +1,997 @@ +/** + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: BUSL-1.1 + */ + +import { expect, test, describe, vi, beforeEach, afterEach } from 'vitest' +import { fs, vol } from 'memfs' +import { buildMdxTransforms } from './build-mdx-transforms.mjs' +import * as repoConfig from '#productConfig.mjs' + +vi.mock('node:fs') +vi.mock('node:fs/promises') + +describe('applyMdxTransforms - Integration Tests', () => { + const mockVersionMetadata = { + vault: [ + { version: 'v1.19.x', releaseStage: 'stable', isLatest: false }, + { version: 'v1.20.x', releaseStage: 'stable', isLatest: false }, + { version: 'v1.21.x', releaseStage: 'stable', isLatest: false }, + { version: 'v1.22.x', releaseStage: 'stable', isLatest: true }, + ], + 'terraform-docs-common': [ + { version: 'v1.20.x', releaseStage: 'stable', isLatest: true }, + ], + 'terraform-enterprise': [ + { version: 'v202409-2', releaseStage: 'stable', isLatest: true }, + ], + } + + beforeEach(() => { + vol.reset() + vi.clearAllMocks() + }) + + afterEach(() => { + vol.reset() + }) + + describe('Global Partials Processing', () => { + test('should process global partials and include content in output', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + vault: { + versionedDocs: true, + basePaths: ['docs'], + supportsExclusionDirectives: false, + }, + }) + + const globalPartialContent = `--- +page_title: Global Partial +--- + +This is global partial content that should be included.` + + const mainContent = `--- +page_title: Test Page +--- + +# Test Page + +@include 'global-test.mdx' + +Regular content after partial. +` + + vol.fromJSON({ + '/content/vault/v1.20.x/docs/test.mdx': mainContent, + '/content/vault/v1.20.x/docs/partials/global-test.mdx': + globalPartialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const outputContent = fs.readFileSync( + '/output/vault/v1.20.x/docs/test.mdx', + 'utf8', + ) + expect(outputContent).toContain( + 'This is global partial content that should be included', + ) + expect(outputContent).toContain('Regular content after partial') + expect(outputContent).not.toContain('@include') + }) + + test('should process nested global partials correctly', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + vault: { + versionedDocs: true, + basePaths: ['docs'], + supportsExclusionDirectives: false, + }, + }) + + const nestedPartialContent = `--- +page_title: Nested Partial +--- + +Nested partial content.` + + const parentPartialContent = `--- +page_title: Parent Partial +--- + +Parent partial start. + +@include 'nested.mdx' + +Parent partial end.` + + const mainContent = `--- +page_title: Test Page +--- + +# Test Page + +@include 'parent.mdx' + +Main content. +` + + vol.fromJSON({ + '/content/vault/v1.20.x/docs/test.mdx': mainContent, + '/content/vault/v1.20.x/docs/partials/parent.mdx': parentPartialContent, + '/content/vault/v1.20.x/docs/partials/nested.mdx': nestedPartialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const outputContent = fs.readFileSync( + '/output/vault/v1.20.x/docs/test.mdx', + 'utf8', + ) + expect(outputContent).toContain('Parent partial start') + expect(outputContent).toContain('Nested partial content') + expect(outputContent).toContain('Parent partial end') + expect(outputContent).toContain('Main content') + }) + }) + + describe('Content Exclusion After Partials', () => { + test('should process exclusion directives in global partials', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + vault: { + versionedDocs: true, + basePaths: ['docs'], + supportsExclusionDirectives: true, + }, + }) + + const globalPartialContent = `--- +page_title: Versioned Content +--- + +This content is always visible. + + +This content is only for v1.21.x and later. + + +This content is also always visible.` + + const mainContent = `--- +page_title: Test Page +--- + +# Test Page + +@include 'versioned-content.mdx' +` + + // Test with v1.20.x - should exclude v1.21.x content + vol.fromJSON({ + '/content/vault/v1.20.x/docs/test.mdx': mainContent, + '/content/vault/v1.20.x/docs/partials/versioned-content.mdx': + globalPartialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const outputV120 = fs.readFileSync( + '/output/vault/v1.20.x/docs/test.mdx', + 'utf8', + ) + expect(outputV120).toContain('This content is always visible') + expect(outputV120).not.toContain( + 'This content is only for v1.21.x and later', + ) + expect(outputV120).toContain('This content is also always visible') + + // Reset and test with v1.21.x - should include v1.21.x content + vol.reset() + vol.fromJSON({ + '/content/vault/v1.21.x/docs/test.mdx': mainContent, + '/content/vault/v1.21.x/docs/partials/versioned-content.mdx': + globalPartialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const outputV121 = fs.readFileSync( + '/output/vault/v1.21.x/docs/test.mdx', + 'utf8', + ) + expect(outputV121).toContain('This content is always visible') + expect(outputV121).toContain('This content is only for v1.21.x and later') + expect(outputV121).toContain('This content is also always visible') + }) + + test('should process exclusion directives in main file with included partials', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + vault: { + versionedDocs: true, + basePaths: ['docs'], + supportsExclusionDirectives: true, + }, + }) + + const globalPartialContent = `--- +page_title: New Feature +--- + +Partial content here.` + + const mainContent = `--- +page_title: Test Page +--- + +# Test Page + + +@include 'new-feature.mdx' + + +Regular content. +` + + // Test with v1.20.x - should exclude entire block including partial + vol.fromJSON({ + '/content/vault/v1.20.x/docs/test.mdx': mainContent, + '/content/vault/v1.20.x/docs/partials/new-feature.mdx': + globalPartialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const outputV120 = fs.readFileSync( + '/output/vault/v1.20.x/docs/test.mdx', + 'utf8', + ) + expect(outputV120).not.toContain('Partial content here') + expect(outputV120).toContain('Regular content') + + // Test with v1.21.x - should include partial + vol.reset() + vol.fromJSON({ + '/content/vault/v1.21.x/docs/test.mdx': mainContent, + '/content/vault/v1.21.x/docs/partials/new-feature.mdx': + globalPartialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const outputV121 = fs.readFileSync( + '/output/vault/v1.21.x/docs/test.mdx', + 'utf8', + ) + expect(outputV121).toContain('Partial content here') + expect(outputV121).toContain('Regular content') + }) + + test('should handle multiple exclusion blocks in same partial', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + vault: { + versionedDocs: true, + basePaths: ['docs'], + supportsExclusionDirectives: true, + }, + }) + + const globalPartialContent = `--- +page_title: Complex Partial +--- + +Partial header. + + +This is v1.21.x specific content. + + +Middle content. + + +This is v1.22.x specific content. + + +Partial footer.` + + const mainContent = `--- +page_title: Test Page +--- + +# Test Page + +@include 'complex.mdx' + +Always visible content. +` + + // Test with v1.20.x - should exclude both version-specific blocks + vol.fromJSON({ + '/content/vault/v1.20.x/docs/test.mdx': mainContent, + '/content/vault/v1.20.x/docs/partials/complex.mdx': + globalPartialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const outputV120 = fs.readFileSync( + '/output/vault/v1.20.x/docs/test.mdx', + 'utf8', + ) + expect(outputV120).toContain('Partial header') + expect(outputV120).not.toContain('This is v1.21.x specific content') + expect(outputV120).toContain('Middle content') + expect(outputV120).not.toContain('This is v1.22.x specific content') + expect(outputV120).toContain('Partial footer') + expect(outputV120).toContain('Always visible content') + + // Test with v1.21.x - should include v1.21.x content but exclude v1.22.x + vol.reset() + vol.fromJSON({ + '/content/vault/v1.21.x/docs/test.mdx': mainContent, + '/content/vault/v1.21.x/docs/partials/complex.mdx': + globalPartialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const outputV121 = fs.readFileSync( + '/output/vault/v1.21.x/docs/test.mdx', + 'utf8', + ) + expect(outputV121).toContain('Partial header') + expect(outputV121).toContain('This is v1.21.x specific content') + expect(outputV121).toContain('Middle content') + expect(outputV121).not.toContain('This is v1.22.x specific content') + expect(outputV121).toContain('Partial footer') + expect(outputV121).toContain('Always visible content') + + // Test with v1.22.x - should include all content + vol.reset() + vol.fromJSON({ + '/content/vault/v1.22.x/docs/test.mdx': mainContent, + '/content/vault/v1.22.x/docs/partials/complex.mdx': + globalPartialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const outputV122 = fs.readFileSync( + '/output/vault/v1.22.x/docs/test.mdx', + 'utf8', + ) + expect(outputV122).toContain('Partial header') + expect(outputV122).toContain('This is v1.21.x specific content') + expect(outputV122).toContain('Middle content') + expect(outputV122).toContain('This is v1.22.x specific content') + expect(outputV122).toContain('Partial footer') + expect(outputV122).toContain('Always visible content') + }) + }) + + describe('Error Cases', () => { + test('should handle error when partial file does not exist', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + vault: { + versionedDocs: true, + basePaths: ['docs'], + supportsExclusionDirectives: false, + }, + }) + + const mainContent = `--- +page_title: Test Page +--- + +# Test Page + +@include 'nonexistent.mdx' +` + + vol.fromJSON({ + '/content/vault/v1.20.x/docs/test.mdx': mainContent, + }) + + // Mock console.error to suppress error output during test + const consoleErrorSpy = vi + .spyOn(console, 'error') + .mockImplementation(() => {}) + const processExitSpy = vi + .spyOn(process, 'exit') + .mockImplementation(() => {}) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + expect(processExitSpy).toHaveBeenCalledWith(1) + expect(consoleErrorSpy).toHaveBeenCalled() + + consoleErrorSpy.mockRestore() + processExitSpy.mockRestore() + }) + + test('should handle error with malformed exclusion directive in partial', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + vault: { + versionedDocs: true, + basePaths: ['docs'], + supportsExclusionDirectives: true, + }, + }) + + const globalPartialContent = `--- +page_title: Bad Directive +--- + + +This has an invalid directive. +` + + const mainContent = `--- +page_title: Test Page +--- + +@include 'bad-directive.mdx' +` + + vol.fromJSON({ + '/content/vault/v1.20.x/docs/test.mdx': mainContent, + '/content/vault/v1.20.x/docs/partials/bad-directive.mdx': + globalPartialContent, + }) + + const consoleErrorSpy = vi + .spyOn(console, 'error') + .mockImplementation(() => {}) + const processExitSpy = vi + .spyOn(process, 'exit') + .mockImplementation(() => {}) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + expect(processExitSpy).toHaveBeenCalledWith(1) + expect(consoleErrorSpy).toHaveBeenCalled() + + consoleErrorSpy.mockRestore() + processExitSpy.mockRestore() + }) + + test('should handle error with mismatched exclusion BEGIN/END in partial', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + vault: { + versionedDocs: true, + basePaths: ['docs'], + supportsExclusionDirectives: true, + }, + }) + + const globalPartialContent = `--- +page_title: Mismatched +--- + + +Content here +` + + const mainContent = `--- +page_title: Test Page +--- + +@include 'mismatched.mdx' +` + + vol.fromJSON({ + '/content/vault/v1.20.x/docs/test.mdx': mainContent, + '/content/vault/v1.20.x/docs/partials/mismatched.mdx': + globalPartialContent, + }) + + const consoleErrorSpy = vi + .spyOn(console, 'error') + .mockImplementation(() => {}) + const processExitSpy = vi + .spyOn(process, 'exit') + .mockImplementation(() => {}) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + expect(processExitSpy).toHaveBeenCalledWith(1) + expect(consoleErrorSpy).toHaveBeenCalled() + + consoleErrorSpy.mockRestore() + processExitSpy.mockRestore() + }) + }) + describe('Global Partials Skip Content Exclusion', () => { + test('should skip content exclusion for files in global/partials directory', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + vault: { + versionedDocs: true, + basePaths: ['docs'], + supportsExclusionDirectives: true, + }, + }) + + // Global partial should NOT have exclusion directives processed + // It should remain as-is with the directives intact + const globalPartialContent = `--- +page_title: Global Partial +--- + +This is always visible. + + +This directive should NOT be processed in the global partial file itself. + + +More content.` + + const mainContent = `--- +page_title: Test Page +--- + +# Test Page + +@include '../../../global/partials/mock-global-partial.mdx' +` + + vol.fromJSON({ + '/content/vault/v1.20.x/docs/test-page.mdx': mainContent, + '/content/vault/global/partials/mock-global-partial.mdx': + globalPartialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + // The global partial file itself should NOT be processed for exclusions + // It should be written as-is with directives intact + const globalPartialOutput = fs.readFileSync( + '/output/vault/global/partials/mock-global-partial.mdx', + 'utf8', + ) + expect(globalPartialOutput).toContain('') + expect(globalPartialOutput).toContain( + 'This directive should NOT be processed in the global partial file itself.', + ) + expect(globalPartialOutput).toContain('') + + // However, when included in the main file, the exclusion directives SHOULD be processed + const mainOutput = fs.readFileSync( + '/output/vault/v1.20.x/docs/test-page.mdx', + 'utf8', + ) + expect(mainOutput).toContain('This is always visible') + expect(mainOutput).not.toContain( + 'This directive should NOT be processed in the global partial file itself.', + ) + expect(mainOutput).toContain('More content') + // The directives themselves should be removed from the included content + expect(mainOutput).not.toContain('') + expect(mainOutput).not.toContain('') + }) + }) + + // we don't need cross product support right now since this product doesn't have global partials + // but ill keep it around just in case + describe('Terraform Product Exclusions', () => { + test('should remove TFC:only block wrapping @include in terraform-enterprise', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + 'terraform-enterprise': { + contentDir: 'docs', + versionedDocs: true, + supportsExclusionDirectives: true, + }, + }) + + const partialContent = `-> **Note:** Ephemeral workspace (automatic destroy runs) functionality is available in HCP Terraform **Plus** Edition. Refer to [HCP Terraform pricing](https://www.hashicorp.com/products/terraform/pricing) for details. +` + + const mainContent = `--- +page_title: Managing Projects +--- + +## Automatically destroy inactive workspaces + + + +@include 'tfc-package-callouts/ephemeral-workspaces.mdx' + + + +You can configure HCP Terraform to automatically destroy. +` + + vol.fromJSON({ + '/content/terraform-enterprise/v202409-2/docs/enterprise/projects/managing.mdx': + mainContent, + '/content/terraform-enterprise/v202409-2/docs/partials/tfc-package-callouts/ephemeral-workspaces.mdx': + partialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const output = fs.readFileSync( + '/output/terraform-enterprise/v202409-2/docs/enterprise/projects/managing.mdx', + 'utf8', + ) + + // The TFC:only block should be removed in terraform-enterprise + expect(output).not.toContain('Ephemeral workspace') + expect(output).not.toContain('Plus Edition') + expect(output).toContain('You can configure HCP Terraform') + }) + + test('should process TFC:only directives in global partials', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + 'terraform-docs-common': { + versionedDocs: true, + basePaths: ['docs'], + supportsExclusionDirectives: true, + }, + }) + + const globalPartialContent = `--- +page_title: Product Specific +--- + + +This is TFC-only content. + + + +This is TFE-only content. + + +Common content.` + + const mainContent = `--- +page_title: Test Page +--- + +@include 'product-specific.mdx' +` + + vol.fromJSON({ + '/content/terraform-docs-common/v1.20.x/docs/test.mdx': mainContent, + '/content/terraform-docs-common/v1.20.x/docs/partials/product-specific.mdx': + globalPartialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const output = fs.readFileSync( + '/output/terraform-docs-common/v1.20.x/docs/test.mdx', + 'utf8', + ) + expect(output).toContain('This is TFC-only content') + expect(output).not.toContain('This is TFE-only content') + expect(output).toContain('Common content') + }) + }) + + describe('Multiple Files Processing', () => { + test('should process multiple files with partials and exclusions', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + vault: { + versionedDocs: true, + basePaths: ['docs'], + supportsExclusionDirectives: true, + }, + }) + + const sharedPartial = `--- +page_title: Shared Partial +--- + +Shared partial content.` + + const versionedPartial = `--- +page_title: Versioned Partial +--- + + +New feature documentation. +` + + const file1 = `--- +page_title: File 1 +--- + +@include 'shared.mdx' +` + + const file2 = `--- +page_title: File 2 +--- + +@include 'versioned.mdx' +` + + vol.fromJSON({ + '/content/vault/v1.20.x/docs/file1.mdx': file1, + '/content/vault/v1.20.x/docs/file2.mdx': file2, + '/content/vault/v1.20.x/docs/partials/shared.mdx': sharedPartial, + '/content/vault/v1.20.x/docs/partials/versioned.mdx': versionedPartial, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const output1 = fs.readFileSync( + '/output/vault/v1.20.x/docs/file1.mdx', + 'utf8', + ) + expect(output1).toContain('Shared partial content') + + const output2 = fs.readFileSync( + '/output/vault/v1.20.x/docs/file2.mdx', + 'utf8', + ) + expect(output2).not.toContain('New feature documentation') + }) + + describe('Partial Content Edge Cases - Multi-line Partials', () => { + test('should remove multi-line partial wrapped in TFC:only directive', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + 'terraform-enterprise': { + contentDir: 'docs', + versionedDocs: true, + supportsExclusionDirectives: true, + }, + }) + + // Multi-line partial (2 lines) - this was the bug case + const partialContent = `-> **Note:** Ephemeral workspace (automatic destroy runs) functionality is available in Terraform Cloud **Plus** Edition. Refer to [Terraform Cloud pricing](https://www.hashicorp.com/products/terraform/pric +ing) for details. +` + + const mainContent = `--- +page_title: Notification Configurations +--- + +## Workspace Notifications + + + +@include 'tfc-package-callouts/ephemeral-workspaces.mdx' + + + +Automatic destroy run notifications contain the following information. +` + + vol.fromJSON({ + '/content/terraform-enterprise/v202311-1/docs/enterprise/api-docs/notification-configurations.mdx': + mainContent, + '/content/terraform-enterprise/v202311-1/docs/partials/tfc-package-callouts/ephemeral-workspaces.mdx': + partialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const output = fs.readFileSync( + '/output/terraform-enterprise/v202311-1/docs/enterprise/api-docs/notification-configurations.mdx', + 'utf8', + ) + + // The multi-line partial content should be removed + expect(output).not.toContain('Ephemeral workspace') + expect(output).not.toContain('Plus Edition') + expect(output).not.toContain('pricing') + expect(output).toContain('Automatic destroy run notifications') + }) + + test('should remove partial with multiple paragraphs', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + 'terraform-enterprise': { + contentDir: 'docs', + versionedDocs: true, + supportsExclusionDirectives: true, + }, + }) + + const partialContent = `**Note:** First paragraph. + +Second paragraph with more content. + +Third paragraph. +` + + const mainContent = `--- +page_title: Test +--- + +## Section + + + +@include 'multi-para.mdx' + + + +Keep this content. +` + + vol.fromJSON({ + '/content/terraform-enterprise/v202402-1/docs/test.mdx': mainContent, + '/content/terraform-enterprise/v202402-1/docs/partials/multi-para.mdx': + partialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const output = fs.readFileSync( + '/output/terraform-enterprise/v202402-1/docs/test.mdx', + 'utf8', + ) + + expect(output).not.toContain('First paragraph') + expect(output).not.toContain('Second paragraph') + expect(output).not.toContain('Third paragraph') + expect(output).toContain('Keep this content') + }) + + test('should remove partial with nested markdown (lists, code blocks)', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + 'terraform-enterprise': { + contentDir: 'docs', + versionedDocs: true, + supportsExclusionDirectives: true, + }, + }) + + const partialContent = `**Features:** + +- Item 1 +- Item 2 +- Item 3 + +\`\`\`bash +echo "test" +\`\`\` +` + + const mainContent = `--- +page_title: Test +--- + + + +@include 'nested.mdx' + + + +Regular content. +` + + vol.fromJSON({ + '/content/terraform-enterprise/v202310-1/docs/test.mdx': mainContent, + '/content/terraform-enterprise/v202310-1/docs/partials/nested.mdx': + partialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const output = fs.readFileSync( + '/output/terraform-enterprise/v202310-1/docs/test.mdx', + 'utf8', + ) + + expect(output).not.toContain('Features:') + expect(output).not.toContain('Item 1') + expect(output).not.toContain('echo "test"') + expect(output).toContain('Regular content') + }) + + test('should handle partial with many lines (high line numbers)', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + 'terraform-enterprise': { + contentDir: 'docs', + versionedDocs: true, + supportsExclusionDirectives: true, + }, + }) + + // Partial spanning 20+ lines + const partialContent = `Line 1 content. + +Line 3 content. + +Line 5 content. + +Line 7 content. + +Line 9 content. + +Line 11 content. + +Line 13 content. + +Line 15 content. + +Line 17 content. + +Line 19 content. +` + + const mainContent = `--- +page_title: Test +--- + +## Section + + + +@include 'long.mdx' + + + +Keep this. +` + + vol.fromJSON({ + '/content/terraform-enterprise/v202301-1/docs/test.mdx': mainContent, + '/content/terraform-enterprise/v202301-1/docs/partials/long.mdx': + partialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const output = fs.readFileSync( + '/output/terraform-enterprise/v202301-1/docs/test.mdx', + 'utf8', + ) + + expect(output).not.toContain('Line 1 content') + expect(output).not.toContain('Line 9 content') + expect(output).not.toContain('Line 19 content') + expect(output).toContain('Keep this') + }) + + test('should keep multi-line partial in terraform-docs-common when wrapped in TFC:only', async () => { + vi.spyOn(repoConfig, 'PRODUCT_CONFIG', 'get').mockReturnValue({ + 'terraform-docs-common': { + versionedDocs: true, + basePaths: ['docs'], + supportsExclusionDirectives: true, + }, + }) + + const partialContent = `-> **Note:** Multi-line +partial content for TFC. +` + + const mainContent = `--- +page_title: Test +--- + + + +@include 'tfc-feature.mdx' + + + +Regular content. +` + + vol.fromJSON({ + '/content/terraform-docs-common/v1.9.x/docs/cloud-docs/test.mdx': + mainContent, + '/content/terraform-docs-common/v1.9.x/docs/partials/tfc-feature.mdx': + partialContent, + }) + + await buildMdxTransforms('/content', '/output', mockVersionMetadata) + + const output = fs.readFileSync( + '/output/terraform-docs-common/v1.9.x/docs/cloud-docs/test.mdx', + 'utf8', + ) + + // TFC:only content should be KEPT in terraform-docs-common + expect(output).toContain('Multi-line') + expect(output).toContain('partial content for TFC') + expect(output).toContain('Regular content') + }) + }) + }) +}) diff --git a/scripts/prebuild/mdx-transforms/exclude-content/README.md b/scripts/prebuild/mdx-transforms/exclude-content/README.md new file mode 100644 index 0000000000..9030f98634 --- /dev/null +++ b/scripts/prebuild/mdx-transforms/exclude-content/README.md @@ -0,0 +1,547 @@ +# Content Exclusion Transform + +A single-pass MDX transform that handles content exclusion directives for HashiCorp products with explicit, maintainable if-block routing. + +## How It Works + +### Overview +This transform processes HTML-style comments in MDX files to conditionally include or exclude content based on product and version criteria. It uses a single AST traversal with explicit if-block routing. + +### Directive Format +Content exclusion blocks follow this format: +```html + +Content to conditionally include/exclude + +``` + +### Supported Directive Types + +#### 1. Version Directives (Vault) +```html + +This content appears only in Vault v1.21.x and later + +``` + +**Supported operators:** `>=`, `<=`, `>`, `<`, `=` +**Version format:** `vX.Y.x` (e.g., `v1.20.x`) + +#### 2. "Only" Directives (Terraform products) +```html + +This content appears only in Terraform Cloud docs + + + +This content appears only in Terraform Enterprise docs + +``` + +**Optional name parameter:** +```html + +Content with a descriptive name for documentation purposes + +``` +The `name:` parameter is optional and can be used to add semantic meaning to directive blocks. It does not affect the processing logic - blocks are still evaluated based on the product (TFC/TFEnterprise) and the "only" directive. + +### Cross-Product Behavior + +| Product | TFC:only | TFEnterprise:only | Vault:* | +|---------|----------|-------------------|---------| +| `terraform-docs-common` | Keep | Remove | Ignore | +| `terraform-enterprise` | Remove | Keep | Ignore | +| `terraform` | Remove | Remove | Ignore | +| `vault` | Ignore | Ignore | Process | + +**Legend:** +- **Keep**: Content remains in output +- **Remove**: Content is removed from output +- **Ignore**: Directive blocks are not handled +- **Process**: Apply version comparison logic + +## Architecture + +### File Structure +``` +exclude-content/ +├── index.mjs # Main transform with if-block routing +├── ast-utils.mjs # Block parsing and node removal utilities +├── vault-processor.mjs # Vault version directive processing +├── terraform-processor.mjs # TFC/TFEnterprise only directive processing +├── index.test.mjs # Comprehensive tests +└── README.md # This file +``` + +### Code Flow + +1. **Early Return**: If `productConfig.supportsExclusionDirectives` is false, skip processing +2. **Single AST Pass**: Parse all directive blocks in one traversal (`parseDirectiveBlocks`) +3. **Explicit Routing**: For each block, check if in (`directiveProcessingFuncs`) object to route to appropriate processor: + ```javascript + const directiveProcessingFuncs = { + Vault: processVaultBlock, + TFC: processTFCBlock, + TFEnterprise: processTFEnterpriseBlock, + } + + // Explicit routing + if (product in directiveProcessingFuncs) { + directiveProcessingFuncs[product](directive, block, tree, options) + } + else { + // Error for unknown products + throw new Error( + `Unknown directive product: "${product}" in block "${block.content}" at lines ${block.start}-${block.end}. Expected: Vault, TFC, or TFEnterprise`, + ) + } + ``` +4. **Product-Specific Processing**: Each processor handles its own business logic +5. **Error Handling**: Contextual error messages with line numbers + +### Key Design Principles + +- **Explicit over Implicit**: No configuration-driven pattern matching +- **Single Pass Performance**: Parse all blocks once, route individually +- **Clear Error Messages**: Immediate feedback with file context and line numbers +- **Extensible**: Add new products with an additional key-val pair in directiveProcessingFuncs and one processor file + +## Integration + +### Processing Pipeline Order + +**IMPORTANT**: This transform runs AFTER the `remarkIncludePartialsPlugin` in the MDX processing pipeline: + +```javascript +.use(remarkIncludePartialsPlugin, { partialsDir, filePath }) // ← First: expand all @include statements +.use(transformExcludeContent, { ... }) // ← Second: process exclusion directives +``` + +**Why this order matters (The Chicken/Egg Problem):** + +There's a dependency relationship that dictates this processing order: +- **Content exclusion** needs to see all directive blocks in the AST to process them +- **Global partials** can contain exclusion directives, but aren't in the AST until `@include` statements are expanded +- **Therefore**: If content exclusion runs first, it won't see directives in global partials because they haven't been included yet + +This creates a chicken/egg problem if we try to reverse the order: +1. ❌ If content exclusion runs first: Global partial directives are missed (partials not yet expanded) +2. ✅ If partials run first: All directives are present in the AST (partials are expanded), then content exclusion processes everything + +**How it works:** +- Global partials (e.g., `/vault/global/partials/`) can contain exclusion directives +- The partials plugin expands all `@include` statements into the main AST +- Content exclusion then processes the fully expanded AST with all partial content included +- This ensures exclusion directives in global partials are properly evaluated + +### Global Partials Exception + +**IMPORTANT**: Files located in `*/global/partials/` directories are **excluded** from content exclusion processing: + +```javascript +const isGlobalPartial = filePath.includes('/global/partials/') + +// Only apply content exclusion if this is NOT a global partial +if (!isGlobalPartial) { + processor.use(transformExcludeContent, { ... }) +} +``` + +**Why this matters:** +- Global partials are version-agnostic and shared across all versions +- They should contain the raw exclusion directives, not have them processed +- When a version-specific file includes a global partial, the directives ARE processed based on that file's version +- This allows global partials to contain version-specific content through directives + +**Example:** +1. `/vault/global/partials/feature.mdx` contains `` directive +2. This file is processed WITHOUT content exclusion - directive stays intact +3. `/vault/v1.20.x/docs/page.mdx` includes the global partial via `@include` +4. During processing, the partial is expanded into the v1.20.x file's AST +5. Content exclusion runs on the v1.20.x file, processing the directive (v1.20.x < v1.21.x, content removed) +6. `/vault/v1.21.x/docs/page.mdx` includes the same global partial +7. Content exclusion runs on the v1.21.x file (v1.21.x >= v1.21.x, content kept) + +### Edge Case: Exclusion Directives Wrapping @include Statements + +**The Problem:** + +When an exclusion directive wraps an `@include` statement, there's a subtle bug related to AST position data: + +```html + +@include 'tfc-feature.mdx' + +``` + +**What Should Happen:** +In terraform-enterprise files, the entire block (including the partial content) should be removed. + +**What Was Happening:** +- ✅ BEGIN comment removed (line 65) +- ❌ Partial content NOT removed (reports position as line 1) +- ✅ END comment removed (line 69) +- Result: The partial content survived! + +**Root Cause - AST Position Mismatch:** + +1. The include-partials plugin replaces `@include` with the partial's AST nodes +2. Those nodes retain position data from the **partial file** (line 1, line 2, etc.) +3. Content exclusion's `removeNodesInRange(tree, 65, 69)` looks for nodes between lines 65-69 +4. Partial nodes with `position.start.line = 1` are **not** in range 65-69 +5. They don't get removed + +**Example AST Structure After Partials:** +```javascript +// Parent file: lines 65-69 contain the exclusion block +{ + type: 'html', // BEGIN comment + position: { line: 65 } // ✅ In range, removed +} +{ + type: 'paragraph', // Content from partial + position: { line: 1 } // ❌ NOT in range 65-69, survives +} +{ + type: 'html', // END comment + position: { line: 69 } // ✅ In range, removed +} +``` + +**The Fix:** + +Modified `removeNodesInRange` in `ast-utils.mjs` to track when we're "inside" a removal range: + +```javascript +let insideRange = false + +for (let i = 0; i < nodes.length; i++) { + const node = nodes[i] + + if (hasPosition) { + // Mark when we enter the range (BEGIN comment) + if (nodeStart >= startLine && nodeEnd <= endLine && !insideRange) { + insideRange = true + } + + // Normal case: node fully in range + if (nodeStart >= startLine && nodeEnd <= endLine) { + indicesToRemove.push(i) + } + // Edge case: inside range but position is outside (partial node) + else if (insideRange && nodeStart < startLine) { + indicesToRemove.push(i) // Remove it anyway + } + + // Mark when we exit the range (END comment) + if (nodeEnd === endLine) { + insideRange = false + } + } else { + // Node without position - remove if inside range + if (insideRange) { + indicesToRemove.push(i) + } + } +} +``` + +**Key Logic:** +- If we're between BEGIN and END (`insideRange = true`) +- And a node has position data that doesn't match the expected range (like line 1 when range is 65-69) +- It's a partial node that needs to be removed + +**Testing:** + +Added integration test in `build-mdx-transforms.test.mjs`: +```javascript +test('should remove TFC:only block wrapping @include in terraform-enterprise') +``` + +This test creates the exact scenario: TFC:only wrapping an @include statement in a terraform-enterprise file, and verifies the partial content is removed. + +### build-mdx-transforms.mjs Integration +```javascript +import { transformExcludeContent } from './exclude-content/index.mjs' + +// Check if file is a global partial +const isGlobalPartial = filePath.includes('/global/partials/') + +const processor = remark() + .use(remarkMdx) + .use(remarkIncludePartialsPlugin, { partialsDir, filePath }) + +// Only apply content exclusion if NOT a global partial +if (!isGlobalPartial) { + processor.use(transformExcludeContent, { + filePath, // Full file path + version, // Content version + repoSlug: entry.repoSlug, // Product slug (e.g., 'vault') + productConfig: PRODUCT_CONFIG[entry.repoSlug] // Full product config + }) +} +``` + +### Product Configuration (productConfig.mjs) +```javascript +export const PRODUCT_CONFIG = { + 'vault': { + // ... existing config + supportsExclusionDirectives: true, + }, + 'terraform-docs-common': { + // ... existing config + supportsExclusionDirectives: true, + } +} +``` + +## Adding a New Product for Content Exclusion + +Follow these steps to add content exclusion support for a new product: + +### Step 1: Update Product Configuration +In `productConfig.mjs`, add exclusion support to your product: + +```javascript +export const PRODUCT_CONFIG = { + // ... existing products + 'consul': { + // ... existing consul config + supportsExclusionDirectives: true, + } +} +``` + +### Step 2: Add Routing Logic +In `exclude-content/index.mjs`, add your product to the if-block routing: + +```javascript +const directiveProcessingFuncs = { + Vault: processVaultBlock, + TFC: processTFCBlock, + TFEnterprise: processTFEnterpriseBlock, + Consul: processConsulBlock // <- ADD THIS +} + +// Explicit routing +if (product in directiveProcessingFuncs) { + directiveProcessingFuncs[product](directive, block, tree, options) +} +else { + // Error for unknown products + throw new Error( + `Unknown directive product: "${product}" in block "${block.content}" at lines ${block.start}-${block.end}. Expected: Vault, TFC, TFEnterprise, or Consul`, // <- ADD THIS + ) +} +``` + +### Step 3: Create Product Processor +Create a new file `exclude-content/consul-processor.mjs`: + +```javascript +/** + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: BUSL-1.1 + */ + +import { removeNodesInRange } from './ast-utils.mjs' + +/** + * Process Consul-specific directive blocks + * + * @param {string} directive The directive part (e.g., "only" or ">=v1.15.x") + * @param {Object} block Block information with start, end, content + * @param {Object} tree Remark AST + * @param {Object} options Processing options + */ +export function processConsulBlock(directive, block, tree, options) { + const { repoSlug } = options + + // Option A: "Only" directive (like Terraform products) + if (directive === 'only') { + // Consul:only should be kept ONLY in consul files, removed elsewhere + if (repoSlug !== 'consul') { + removeNodesInRange(tree, block.start, block.end) + } + return + } + + // Option B: Version directive (like Vault) + const versionMatch = directive.match(/^(<=|>=|<|>|=)v(\d+\.\d+\.x)$/) + if (versionMatch) { + // Only process version directives in consul files + if (repoSlug !== 'consul') { + return // Skip - ignore consul version directives in non-consul files + } + + processConsulVersionDirective(versionMatch, block, tree, options) + return + } + + // Invalid directive + throw new Error( + `Invalid Consul directive: "${directive}" at lines ${block.start}-${block.end}. ` + + `Expected format: Consul:only or Consul:>=vX.Y.x` + ) +} + +/** + * Process Consul version directives (if needed) + */ +function processConsulVersionDirective(versionMatch, block, tree, options) { + // Implementation similar to vault-processor.mjs if version directives are needed + // ... version comparison logic +} +``` + +### Step 4: Import the Processor +In `exclude-content/index.mjs`, add the import: + +```javascript +import { parseDirectiveBlocks } from './ast-utils.mjs' +import { processVaultBlock } from './vault-processor.mjs' +import { processTFCBlock, processTFEnterpriseBlock } from './terraform-processor.mjs' +import { processConsulBlock } from './consul-processor.mjs' // ← ADD THIS +``` + +### Step 5: Add Tests +In `exclude-content/index.test.mjs`, add test cases for your new product: + +```javascript +describe('transformExcludeContent - Consul Directives', () => { + const consulOptions = { + filePath: 'consul/some-file.md', + version: '1.15.x', + repoSlug: 'consul', + productConfig: { + supportsExclusionDirectives: true, + } + } + + it('should keep Consul:only content in consul files', async () => { + const markdown = ` + +This consul content should stay. + +Regular content. +` + const result = await runTransform(markdown, consulOptions) + expect(result.trim()).toContain('This consul content should stay.') + }) + + it('should remove Consul:only content from non-consul files', async () => { + const nonConsulOptions = { + filePath: 'vault/some-file.md', + repoSlug: 'vault', + productConfig: { supportsExclusionDirectives: true } + } + + const markdown = ` + +This should be removed. + +Regular content. +` + const result = await runTransform(markdown, nonConsulOptions) + expect(result.trim()).toBe('Regular content.') + }) +}) +``` + +### Step 6: Update Documentation +Update this README to include your new product in: +- The supported directive types section +- The cross-product behavior table +- The error message examples + +## Example Usage in Documentation + +```html + + +## New Feature in Vault 1.21.x +This feature is only available in Vault 1.21.x and later versions. + + + + +Click the **Settings** tab in the Terraform Cloud UI. + + + + +Navigate to `/admin/settings` in your Terraform Enterprise instance. + +``` + +## Error Handling + +The transform provides clear, contextual error messages: + +``` +Unknown directive product: "InvalidProduct" in block "InvalidProduct:only" at lines 5-7. Expected: Vault, TFC, or TFEnterprise + +Mismatched block names: BEGIN="Vault:>=v1.21.x" at line 3, END="Vault:>=v1.22.x" at line 5 + +Invalid Vault directive: "invalidformat" at lines 8-10. Expected format: Vault:>=vX.Y.x +``` + +## Performance Considerations + +- **Single AST Traversal**: All directive blocks are parsed in one pass +- **Reverse Processing**: Blocks are processed in reverse order for safe node removal +- **Early Returns**: Products without exclusion support skip processing entirely + +## Testing + +### Unit Tests + +Run the unit tests with: +```bash +npx vitest scripts/prebuild/mdx-transforms/exclude-content +``` + +The unit test suite (`index.test.mjs`) covers: +- Version directive processing for Vault +- "Only" directive processing for Terraform products +- Cross-product behavior (ignore vs remove vs keep) +- Error handling for malformed directives +- Configuration edge cases + +### Integration Tests + +Run the integration tests with: +```bash +npx vitest scripts/prebuild/mdx-transforms/build-mdx-transforms.test.mjs +``` + +The integration test suite (`build-mdx-transforms.test.mjs`) covers the full MDX processing pipeline: + +**Global Partials Processing:** +- Basic partial inclusion and content expansion +- Nested partials (partials that include other partials) + +**Content Exclusion After Partials:** +- Exclusion directives inside global partials work correctly +- Exclusion directives wrapping `@include` statements +- Multiple exclusion blocks in the same partial + +**Global Partials Skip Logic:** +- Files in `*/global/partials/` directories skip content exclusion +- Global partial files retain their directives +- Directives are processed when partials are included in version-specific files + +**Error Cases:** +- Missing partial files +- Malformed exclusion directives in partials +- Mismatched BEGIN/END tags in partials + +**Cross-Product Support:** +- TFC:only and TFEnterprise:only directives in global partials + +**Multi-File Processing:** +- Processing multiple files with shared and versioned partials + +All tests use mock filesystem data (`memfs`) and do not rely on real files in the repository. \ No newline at end of file diff --git a/scripts/prebuild/mdx-transforms/exclude-content/ast-utils.mjs b/scripts/prebuild/mdx-transforms/exclude-content/ast-utils.mjs new file mode 100644 index 0000000000..c42f2a80ae --- /dev/null +++ b/scripts/prebuild/mdx-transforms/exclude-content/ast-utils.mjs @@ -0,0 +1,160 @@ +/** + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: BUSL-1.1 + */ + +import visit from 'unist-util-visit' + +// Regex patterns for BEGIN/END comments +const BEGIN_RE = /^(\s+)?(\s+)?$/ +const END_RE = /^(\s+)?(\s+)?$/ + +/** + * Parse all directive blocks from AST in a single pass + * Simple, explicit error handling for malformed blocks + * + * @param {Object} tree Remark AST + * @returns {Array} Array of block objects with start, end, and content + */ +export function parseDirectiveBlocks(tree) { + const blocks = [] + let currentBlock = null + + visit(tree, (node) => { + const nodeValue = node.value + const lineNumber = node.position?.end?.line + + if (!nodeValue || !lineNumber) { + return + } + + // Handle BEGIN blocks + const beginMatch = nodeValue.match(BEGIN_RE) + if (beginMatch) { + if (currentBlock) { + throw new Error( + `Nested BEGIN blocks not allowed. Found BEGIN at line ${lineNumber}, previous BEGIN at line ${currentBlock.start}`, + ) + } + + const blockContent = beginMatch.groups?.block + if (!blockContent?.trim()) { + throw new Error(`Empty BEGIN block at line ${lineNumber}`) + } + + currentBlock = { + start: lineNumber, + content: blockContent.trim(), + end: null, + } + return + } + + // Handle END blocks + const endMatch = nodeValue.match(END_RE) + if (endMatch) { + if (!currentBlock) { + throw new Error( + `Unexpected END block at line ${lineNumber}. No matching BEGIN block found`, + ) + } + + const endContent = endMatch.groups?.block + if (!endContent?.trim()) { + throw new Error(`Empty END block at line ${lineNumber}`) + } + + if (endContent.trim() !== currentBlock.content) { + throw new Error( + `Mismatched block names: BEGIN="${currentBlock.content}" at line ${currentBlock.start}, ` + + `END="${endContent.trim()}" at line ${lineNumber}`, + ) + } + + // Complete the block + currentBlock.end = lineNumber + blocks.push(currentBlock) + currentBlock = null + } + }) + + // Check for unclosed blocks + if (currentBlock) { + throw new Error( + `Unclosed BEGIN block: "${currentBlock.content}" opened at line ${currentBlock.start}`, + ) + } + + return blocks +} + +/** + * Remove nodes from AST within specified line range + * + * This handles nodes from included partials which may not have position data + * matching the parent file's line numbers. We track when we enter/exit the + * removal range based on the BEGIN/END comments. + * + * @param {Object} tree Remark AST + * @param {number} startLine Start line (inclusive) + * @param {number} endLine End line (inclusive) + */ +export function removeNodesInRange(tree, startLine, endLine) { + function removeFromNodes(nodes) { + if (!Array.isArray(nodes)) { + return + } + + const indicesToRemove = [] + let insideRange = false + + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i] + const hasPosition = node.position?.start?.line && node.position?.end?.line + + if (hasPosition) { + const nodeStart = node.position.start.line + const nodeEnd = node.position.end.line + + // Check if this node marks the start of the range + if (nodeStart >= startLine && nodeEnd <= endLine && !insideRange) { + insideRange = true + } + + // If node is fully within range, mark for removal + if (nodeStart >= startLine && nodeEnd <= endLine) { + indicesToRemove.push(i) + } + // If we're inside range but node has position that indicates it's from a partial + // (position data doesn't match parent file), remove it + else if (insideRange) { + // Node from partial - has position data from partial file, not parent + indicesToRemove.push(i) + } + + // Check if this node marks the end of the range + if (nodeEnd === endLine) { + insideRange = false + } + } else { + // Node without position (e.g., from included partial) + // Remove it if we're currently inside the range + if (insideRange) { + indicesToRemove.push(i) + } + } + + // Recursively check children for nodes not being removed + if (node.children && !indicesToRemove.includes(i)) { + removeFromNodes(node.children) + } + } + + // Remove marked nodes in reverse order to maintain indices + for (let i = indicesToRemove.length - 1; i >= 0; i--) { + nodes.splice(indicesToRemove[i], 1) + } + } + + removeFromNodes(tree.children) +} diff --git a/scripts/prebuild/mdx-transforms/exclude-content/index.mjs b/scripts/prebuild/mdx-transforms/exclude-content/index.mjs new file mode 100644 index 0000000000..895565a24f --- /dev/null +++ b/scripts/prebuild/mdx-transforms/exclude-content/index.mjs @@ -0,0 +1,75 @@ +/** + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: BUSL-1.1 + */ + +import { parseDirectiveBlocks } from './ast-utils.mjs' +import { processVaultBlock } from './vault-processor.mjs' +import { + processTFCBlock, + processTFEnterpriseBlock, +} from './terraform-processor.mjs' + +/** + * Content exclusion transform with explicit if-block routing + * for each product- intended to run through a single AST pass + * + * @param {Object} options Transform options + * @param {string} options.filePath File path being processed + * @param {string} options.version Content version (for version directives) + * @param {string} options.repoSlug Product repo slug (e.g., 'vault', 'terraform-docs-common') + * @param {Object} options.productConfig Product configuration from PRODUCT_CONFIG + * @returns {Function} Remark transformer function + */ +export function transformExcludeContent(options = {}) { + return function transformer(tree) { + const { productConfig } = options + + // Early return if product doesn't support exclusion directives + if (!productConfig?.supportsExclusionDirectives) { + return tree + } + + try { + // Single AST pass to find all directive blocks + const blocks = parseDirectiveBlocks(tree) + + // Process each block with explicit routing (reverse order for safe removal) + blocks.reverse().forEach((block) => { + routeAndProcessBlock(block, tree, options) + }) + + return tree + } catch (error) { + // Add file context to any errors + throw new Error( + `Content exclusion failed in ${options.filePath}: ${error.message}`, + ) + } + } +} + +/** + * Route directive blocks to appropriate processors with if-blocks + */ +function routeAndProcessBlock(block, tree, options) { + // Parse the directive: "Vault:>=v1.21.x" -> product="Vault", directive=">=v1.21.x" + const [product, ...rest] = block.content.split(':') + const directive = rest.join(':') // Handle edge cases like "TFEnterprise:only name:something" + + const directiveProcessingFuncs = { + Vault: processVaultBlock, + TFC: processTFCBlock, + TFEnterprise: processTFEnterpriseBlock, + } + + // Explicit routing + if (product in directiveProcessingFuncs) { + directiveProcessingFuncs[product](directive, block, tree, options) + } else { + // Error for unknown products + throw new Error( + `Unknown directive product: "${product}" in block "${block.content}" at lines ${block.start}-${block.end}. Expected: Vault, TFC, or TFEnterprise`, + ) + } +} diff --git a/scripts/prebuild/mdx-transforms/exclude-content/index.test.mjs b/scripts/prebuild/mdx-transforms/exclude-content/index.test.mjs new file mode 100644 index 0000000000..9dd1e7500b --- /dev/null +++ b/scripts/prebuild/mdx-transforms/exclude-content/index.test.mjs @@ -0,0 +1,404 @@ +/** + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: BUSL-1.1 + */ + +import { describe, it, expect } from 'vitest' +import { transformExcludeContent } from './index.mjs' +import remark from 'remark' +import remarkMdx from 'remark-mdx' + +const runTransform = async (markdown, options) => { + const processor = await remark() + .use(remarkMdx) + .use(transformExcludeContent, options) + .process(markdown) + return processor.contents +} + +// Mock product configs +const vaultConfig = { + supportsExclusionDirectives: true, +} + +const terraformDocsCommonConfig = { + supportsExclusionDirectives: true, +} + +const terraformEnterpriseConfig = { + supportsExclusionDirectives: true, +} + +const noExclusionConfig = { + supportsExclusionDirectives: undefined, +} + +describe('transformExcludeContent - Vault Directives', () => { + const vaultOptions = { + filePath: 'vault/some-file.md', + version: '1.20.x', + repoSlug: 'vault', + productConfig: vaultConfig, + } + + it('should remove content when version condition is not met', async () => { + const markdown = ` + +This content should be removed. + +This content should stay. +` + const result = await runTransform(markdown, vaultOptions) + expect(result).toBe('This content should stay.\n') + }) + + it('should keep content when version condition is met', async () => { + const markdown = ` + +This content should stay. + +Other content. +` + const result = await runTransform(markdown, vaultOptions) + expect(result.trim()).toBe(` + +This content should stay. + + + +Other content.`) + }) + + it('should handle equality comparisons', async () => { + const equalOptions = { ...vaultOptions, version: '1.20.x' } + const markdown = ` + +This content should stay. + +` + const result = await runTransform(markdown, equalOptions) + expect(result.trim()).toBe(` + +This content should stay. + +`) + }) + + it('should handle inequal comparisons', async () => { + const equalOptions = { ...vaultOptions, version: '1.19.x' } + const markdown = ` + +This content should be removed. + +This content should stay. +` + const result = await runTransform(markdown, equalOptions) + expect(result.trim()).toBe(`This content should stay.`) + }) + + it('should handle less than comparisons', async () => { + const markdown = ` + +This content should be removed. + +` + const result = await runTransform(markdown, vaultOptions) + + expect(result.trim()).toBe('') + }) + + it('should handle multiple version blocks correctly', async () => { + const markdown = ` + +This should be removed. + + +This should stay. + +Final content. +` + const result = await runTransform(markdown, vaultOptions) + + expect(result.trim()).toBe(` + +This should stay. + + + +Final content.`) + }) +}) + +describe('transformExcludeContent - TFC/TFEnterprise Directives', () => { + it('should keep TFC:only content in terraform-docs-common', async () => { + const options = { + filePath: 'terraform-docs-common/cloud-docs/some-file.md', + version: 'v1.20.x', + repoSlug: 'terraform-docs-common', + productConfig: terraformDocsCommonConfig, + } + + const markdown = ` + +This content should NOT be removed. + +This content should stay. +` + const result = await runTransform(markdown, options) + expect(result.trim()).toBe(` + +This content should NOT be removed. + + + +This content should stay.`) + }) + + it('should remove TFC:only content from terraform-enterprise', async () => { + const options = { + filePath: 'terraform-enterprise/some-file.md', + version: 'v1.20.x', + repoSlug: 'terraform-enterprise', + productConfig: terraformEnterpriseConfig, + } + + const markdown = ` + +This content should be removed. + + +This content should NOT be removed. + +This content should stay.` + + const expected = ` + +This content should NOT be removed. + + + +This content should stay. +` + + const result = await runTransform(markdown, options) + expect(result).toBe(expected) + }) + + // This is a good test in case partials are used and write to multiple unintended product directories + it('should remove both TFC:only and TFEnterprise:only from terraform product', async () => { + const options = { + filePath: 'terraform/some-file.md', + version: 'v1.20.x', + repoSlug: 'terraform', + productConfig: { supportsExclusionDirectives: true }, + } + + const markdown = ` + +This content should be removed. + + +This content should be removed. + +This content should stay. +` + + const result = await runTransform(markdown, options) + expect(result.trim()).toBe('This content should stay.') + }) + + // Here adding in test for cross product support- this behavior is currently well documented in the README so if any change needs to happen later + // it can + it('should remove TFEnterprise:only with name parameter from terraform product', async () => { + const options = { + filePath: 'terraform/some-file.md', + version: 'v1.20.x', + repoSlug: 'terraform', + productConfig: { supportsExclusionDirectives: true }, + } + + const markdown = ` + +This content should be removed. + + +- You can now revoke, and revert the revocation of, module versions. Learn more about [Managing module versions](/terraform/enterprise/api-docs/private-registry/manage-module-versions). + + +This content should stay. +` + + const result = await runTransform(markdown, options) + expect(result.trim()).toBe('This content should stay.') + }) + + it('should remove TFC:only with name parameter from terraform-enterprise', async () => { + const options = { + filePath: + 'terraform-enterprise/v202409-2/docs/enterprise/projects/managing.md', + version: 'v202409-2', + repoSlug: 'terraform-enterprise', + productConfig: terraformEnterpriseConfig, + } + + const markdown = ` +## Automatically destroy inactive workspaces + + +**Note:** Ephemeral workspace functionality is available in HCP Terraform Plus Edition. + + +You can configure HCP Terraform to automatically destroy. +` + + const result = await runTransform(markdown, options) + expect(result).not.toContain('Ephemeral workspace') + expect(result).toContain('You can configure HCP Terraform') + }) + + it('should throw an error for mismatched block name directives', async () => { + const options = { + filePath: 'terraform-enterprise/some-file.md', + version: '1.20.x', + repoSlug: 'terraform-enterprise', + productConfig: terraformEnterpriseConfig, + } + const markdown = ` + +This content should be removed. + +` + await expect(async () => { + return await runTransform(markdown, options) + }).rejects.toThrow('Mismatched block names') + }) +}) + +describe('transformExcludeContent - Error Handling', () => { + const vaultOptions = { + filePath: 'vault/some-file.md', + version: '1.20.x', + repoSlug: 'vault', + productConfig: vaultConfig, + } + + it('should throw error for unknown directive products', async () => { + const markdown = ` + +This content should throw an error + +` + await expect(async () => { + return await runTransform(markdown, vaultOptions) + }).rejects.toThrow('Unknown directive product: "INVALID"') + }) + + it('should throw error for mismatched block names', async () => { + const markdown = ` + +This content should be removed. + +` + await expect(async () => { + return await runTransform(markdown, vaultOptions) + }).rejects.toThrow('Mismatched block names') + }) + + it('should throw error for invalid vault directive format', async () => { + const markdown = ` + +This content should throw an error. + +` + await expect(async () => { + return await runTransform(markdown, vaultOptions) + }).rejects.toThrow('Invalid Vault directive: "invalid"') + }) + + it('should throw an error for unexpected END block', async () => { + const markdown = ` + +` + await expect(async () => { + return await runTransform(markdown, vaultOptions) + }).rejects.toThrow('Unexpected END block') + }) + + it('should throw an error for unexpected BEGIN block', async () => { + const markdown = ` + + +` + await expect(async () => { + return await runTransform(markdown, vaultOptions) + }).rejects.toThrow('Nested BEGIN blocks not allowed') + }) + + it('should throw an error if no block could be parsed from BEGIN comment', async () => { + const markdown = ` + +This content should be removed. + +` + await expect(async () => { + return await runTransform(markdown, vaultOptions) + }).rejects.toThrow('Empty BEGIN block') + }) + + it('should throw an error if no block could be parsed from END comment', async () => { + const markdown = ` + +This content should be removed. + +` + await expect(async () => { + return await runTransform(markdown, vaultOptions) + }).rejects.toThrow('Empty END block') + }) +}) + +describe('transformExcludeContent - Configuration', () => { + it('should skip processing when supportsExclusionDirectives is false/undefined', async () => { + const options = { + filePath: 'some-product/some-file.md', + version: 'v1.20.x', + repoSlug: 'some-product', + productConfig: noExclusionConfig, + } + + const markdown = ` + + +This should be ignored. + + + +Content stays. +` + + const result = await runTransform(markdown, options) + expect(result.trim()).toBe(markdown.trim()) + }) + + it('should skip processing when no productConfig provided', async () => { + const options = { + filePath: 'some-product/some-file.md', + version: 'v1.20.x', + repoSlug: 'some-product', + // No productConfig + } + + const markdown = ` + + +This should be ignored. + + + +Content stays. +` + + const result = await runTransform(markdown, options) + expect(result.trim()).toBe(markdown.trim()) + }) +}) diff --git a/scripts/prebuild/mdx-transforms/exclude-content/terraform-processor.mjs b/scripts/prebuild/mdx-transforms/exclude-content/terraform-processor.mjs new file mode 100644 index 0000000000..8174c72b7d --- /dev/null +++ b/scripts/prebuild/mdx-transforms/exclude-content/terraform-processor.mjs @@ -0,0 +1,61 @@ +/** + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: BUSL-1.1 + */ + +import { removeNodesInRange } from './ast-utils.mjs' + +/** + * Process TFC (Terraform Cloud) directive blocks + * Implements cross-product behavior based on your test requirements + * + * @param {string} directive The directive part (e.g., "only") + * @param {Object} block Block information with start, end, content + * @param {Object} tree Remark AST + * @param {Object} options Processing options + */ +export function processTFCBlock(directive, block, tree, options) { + const { repoSlug } = options + + // Handle "only" or "only name:something" format + if (directive === 'only' || directive.startsWith('only ')) { + // TFC:only should be kept ONLY in terraform-docs-common, removed everywhere else + if (repoSlug !== 'terraform-docs-common') { + removeNodesInRange(tree, block.start, block.end) + } + return + } + + // If we get here, it's an invalid TFC directive + throw new Error( + `Invalid TFC directive: "${directive}" at lines ${block.start}-${block.end}. ` + + `Expected format: TFC:only`, + ) +} + +/** + * Process TFEnterprise (Terraform Enterprise) directive blocks + * Implements cross-product behavior based on your test requirements + * + * @param {string} directive The directive part (e.g., "only") + * @param {Object} block Block information with start, end, content + * @param {Object} tree Remark AST + * @param {Object} options Processing options + */ +export function processTFEnterpriseBlock(directive, block, tree, options) { + const { repoSlug } = options + + // Handle "only" and "only name:something" format + if (directive === 'only' || directive.startsWith('only ')) { + // TFEnterprise:only kept only in terraform-enterprise, removed everywhere else + if (repoSlug !== 'terraform-enterprise') { + removeNodesInRange(tree, block.start, block.end) + } + return + } + + // If we get here, it's an invalid TFEnterprise directive + throw new Error( + `Invalid TFEnterprise directive: "${directive}" at lines ${block.start}-${block.end}. Expected format: TFEnterprise:only`, + ) +} diff --git a/scripts/prebuild/mdx-transforms/exclude-content/vault-processor.mjs b/scripts/prebuild/mdx-transforms/exclude-content/vault-processor.mjs new file mode 100644 index 0000000000..389ccc1893 --- /dev/null +++ b/scripts/prebuild/mdx-transforms/exclude-content/vault-processor.mjs @@ -0,0 +1,97 @@ +/** + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: BUSL-1.1 + */ + +import { SemVer, gt, gte, lt, lte, eq } from 'semver' +import { removeNodesInRange } from './ast-utils.mjs' + +/** + * Process Vault-specific directive blocks + * Only processes Vault directives in vault files - ignores them elsewhere + * + * @param {string} directive The directive part (e.g., ">=v1.21.x") + * @param {Object} block Block information with start, end, content + * @param {Object} tree Remark AST + * @param {Object} options Processing options + */ +export function processVaultBlock(directive, block, tree, options) { + const { repoSlug } = options // can pull out { repoSlug, version } if needed + + // Only process Vault blocks in vault files - ignore them elsewhere + if (repoSlug !== 'vault') { + return // Skip vault directive in a non-vault file + } + + // Parse Vault version directive pattern: >=v1.21.x + const versionMatch = directive.match(/^(<=|>=|<|>|=)v(\d+\.\d+\.x)$/) + if (versionMatch) { + processVaultVersionDirective(versionMatch, block, tree, options) + return + } + + // If we get here, it's an invalid Vault directive + throw new Error( + `Invalid Vault directive: "${directive}" at lines ${block.start}-${block.end}. ` + + `Expected format: Vault:>=vX.Y.x`, + ) +} + +/** + * Process Vault version directives (e.g., Vault:>=v1.21.x) + */ +function processVaultVersionDirective(versionMatch, block, tree, options) { + const { version } = options + const [, comparator, directiveVersion] = versionMatch + + if (!version) { + throw new Error( + `Version directive requires version option at lines ${block.start}-${block.end}`, + ) + } + + try { + const currentVersion = normalizeSemver(version) + const targetVersion = normalizeSemver(directiveVersion) + const comparisonFn = getComparisonFunction(comparator) + + const shouldKeepContent = comparisonFn(currentVersion, targetVersion) + + // If version comparison fails, remove the content + if (!shouldKeepContent) { + removeNodesInRange(tree, block.start, block.end) + } + } catch (error) { + throw new Error( + `Version comparison failed for "${block.content}" at lines ${block.start}-${block.end}: ${error.message}`, + ) + } +} + +/** + * Normalize version string for semver comparison + */ +function normalizeSemver(version) { + const normalized = version.replace(/^v/, '').replace(/\.x$/, '.0') + return new SemVer(normalized) +} + +/** + * Get comparison function for operator + */ +function getComparisonFunction(operator) { + const functions = { + '<=': lte, + '>=': gte, + '<': lt, + '>': gt, + '=': eq, + } + + const fn = functions[operator] + if (!fn) { + throw new Error(`Invalid comparison operator: ${operator}`) + } + + return fn +} diff --git a/scripts/prebuild/mdx-transforms/exclude-terraform-content/index.mjs b/scripts/prebuild/mdx-transforms/exclude-terraform-content/index.mjs index b7b30a98fe..ef104909ad 100644 --- a/scripts/prebuild/mdx-transforms/exclude-terraform-content/index.mjs +++ b/scripts/prebuild/mdx-transforms/exclude-terraform-content/index.mjs @@ -4,6 +4,7 @@ */ import visit from 'unist-util-visit' +import { DIRECTIVE_PRODUCTS } from '../shared.mjs' // this is a courtesy wrapper to prepend error messages class ExcludeTerraformContentError extends Error { @@ -21,6 +22,7 @@ export const BEGIN_RE = /^(\s+)?(\s+)?$/ export const END_RE = /^(\s+)?(\s+)?$/ export const DIRECTIVE_RE = /(?TFC|TFEnterprise):only/i +// Adding the directive products parameter to allow for extensibility in tests export function transformExcludeTerraformContent({ filePath }) { return function transformer(tree) { // accumulate the content exclusion blocks @@ -125,6 +127,12 @@ export function transformExcludeTerraformContent({ filePath }) { // TODO: line start and end do not take into account front matter, as it is just tree parsing and technically front matter is not part of the MDX tree if (!directive) { + // Check if this is a product we should handle + const productMatch = flag.match(/^(\w+):/) + + if (productMatch && DIRECTIVE_PRODUCTS.includes(productMatch[1])) { + return // Skip this directive - it's for another product + } throw new ExcludeTerraformContentError( `Directive block ${block} could not be parsed between lines ${start} and ${end}`, tree, diff --git a/scripts/prebuild/mdx-transforms/exclude-terraform-content/index.test.mjs b/scripts/prebuild/mdx-transforms/exclude-terraform-content/index.test.mjs index 8df26fb657..37601db90d 100644 --- a/scripts/prebuild/mdx-transforms/exclude-terraform-content/index.test.mjs +++ b/scripts/prebuild/mdx-transforms/exclude-terraform-content/index.test.mjs @@ -8,6 +8,15 @@ import { transformExcludeTerraformContent } from './index.mjs' import remark from 'remark' import remarkMdx from 'remark-mdx' +// Mock for testing custom directive products +vi.mock('../shared.mjs', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + DIRECTIVE_PRODUCTS: ['Vault', 'TFC', 'TFEnterprise'], // Default for most tests + } +}) + const runTransform = async (markdown, filePath) => { const processor = await remark() .use(remarkMdx) diff --git a/scripts/prebuild/mdx-transforms/exclude-vault-content/index.mjs b/scripts/prebuild/mdx-transforms/exclude-vault-content/index.mjs new file mode 100644 index 0000000000..13d5f65006 --- /dev/null +++ b/scripts/prebuild/mdx-transforms/exclude-vault-content/index.mjs @@ -0,0 +1,242 @@ +/** + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: BUSL-1.1 + */ + +import visit from 'unist-util-visit' +import { SemVer, gt, gte, lt, lte, eq } from 'semver' +import { DIRECTIVE_PRODUCTS } from '../shared.mjs' + +// this is a courtesy wrapper to prepend error messages +class ExcludeVaultContentError extends Error { + constructor(message, markdownSource) { + super( + `[strip-vault-content] ${message}` + + `\n- ${markdownSource}` + + `\n- ${markdownSource}`, + ) + this.name = 'ExcludeVaultContentError' + } +} + +export const BEGIN_RE = /^(\s+)?(\s+)?$/ +export const END_RE = /^(\s+)?(\s+)?$/ +export const DIRECTIVE_RE = + /^(?Vault):(?<=|>=|<|>|=)v(?(\d+)\.(\d+)\.x)$/i + +// Adding the directive products parameter to allow for extensibility in tests +export function transformExcludeVaultContent({ filePath, version }) { + return function transformer(tree) { + // accumulate the content exclusion blocks + /** @type ({ start: number; block: string; end: number })[] */ + const matches = [] + let matching = false + let block = '' + + visit(tree, (node) => { + const nodeValue = node.value + const nodeIndex = node.position?.end?.line + + if (!nodeValue || !nodeIndex || !filePath.includes('vault')) { + return + } + + if (!matching) { + // Wait for a BEGIN block to be matched + + // throw if an END block is matched first + const endMatch = nodeValue.match(END_RE) + if (endMatch) { + throw new ExcludeVaultContentError( + `Unexpected END block: line ${nodeIndex}`, + tree, + ) + } + + const beginMatch = nodeValue.match(BEGIN_RE) + + if (beginMatch) { + matching = true + + if (!beginMatch.groups?.block) { + throw new ExcludeVaultContentError( + 'No block could be parsed from BEGIN comment', + tree, + ) + } + + block = beginMatch.groups.block + + matches.push({ + start: nodeIndex, + block: beginMatch.groups.block, + end: -1, + }) + } + } else { + // If we are actively matching within a block, monitor for the end + + // throw if a BEGIN block is matched again + const beginMatch = nodeValue.match(BEGIN_RE) + if (beginMatch) { + throw new ExcludeVaultContentError( + `Unexpected BEGIN block: line ${nodeIndex}`, + tree, + ) + } + + const endMatch = nodeValue.match(END_RE) + if (endMatch) { + const latestMatch = matches[matches.length - 1] + + if (!endMatch.groups?.block) { + throw new ExcludeVaultContentError( + 'No block could be parsed from END comment', + tree, + ) + } + + // If we reach and end with an un-matching block name, throw an error + if (endMatch.groups.block !== block) { + const errMsg = + `Mismatched block names: Block opens with "${block}", and closes with "${endMatch.groups.block}".` + + `\n` + + `Please make sure opening and closing block names are matching. Blocks cannot be nested.` + + `\n` + + `- Open: ${latestMatch.start}: ${block}` + + `\n` + + `- Close: ${nodeIndex}: ${endMatch.groups.block}` + + `\n` + console.error(errMsg) + throw new ExcludeVaultContentError('Mismatched block names', tree) + } + + // Push the ending index of the block into the match result and set matching to false + latestMatch.end = nodeIndex + block = '' + matching = false + } + } + }) + + // iterate through the list of matches backwards to remove lines + matches.reverse().forEach(({ start, end, block }) => { + const [flag] = block.split(/\s+/) + const directive = flag.match(DIRECTIVE_RE) + + if (!directive?.groups) { + // Check if this is a product we should handle + const productMatch = flag.match(/^(\w+):/) + + // If the product matches the current one we care about 'Vault' then + // continue with further checks on the version and comparator + if (productMatch && productMatch[1] === 'Vault') { + // This is our product, but directive didn't match - check if it's a version format issue + const versionFormatCheck = flag.match(/^(\w+):(<=|>=|<|>|=)v(.+)$/) + + if (versionFormatCheck) { + const [, , , versionPart] = versionFormatCheck + // Check if version format is invalid (not X.Y.x pattern) + if (!versionPart.match(/^\d+\.\d+\.x$/)) { + throw new ExcludeVaultContentError( + `Invalid version format in directive: ${flag}. Expected format: vX.Y.x`, + tree, + ) + } + } + // If we get here, it's some other directive format error + throw new ExcludeVaultContentError( + `Invalid directive format: ${flag}`, + tree, + ) + } + + // otherwise if it is in the directive products list, skip it + else if (productMatch && DIRECTIVE_PRODUCTS.includes(productMatch[1])) { + return // Skip this directive - it's for another product + } + + // else if is not in the product list, throw this error + throw new ExcludeVaultContentError( + `Directive block ${block} could not be parsed between lines ${start} and ${end}`, + tree, + ) + } + + // This is the version that is parsed from reading the filename + const currentVersion = version || '' + + // This is the directive that is parsed from the exclusion block + const { comparator, version: directiveVersion } = directive.groups + + try { + const versionSemVer = getTfeSemver(currentVersion) + const directiveSemVer = getTfeSemver(directiveVersion) + const compare = getComparisonFn(comparator, tree) + + const shouldKeepContent = compare(versionSemVer, directiveSemVer) + + // If the version comparison fails, remove the content + if (!shouldKeepContent) { + function removeNodesInRange(nodes) { + for (let i = nodes.length - 1; i >= 0; i--) { + const node = nodes[i] + if ( + node.position && + node.position.start.line >= start && + node.position.end.line <= end + ) { + nodes.splice(i, 1) + } else if (node.children && Array.isArray(node.children)) { + removeNodesInRange(node.children, node) + } + } + } + removeNodesInRange(tree.children, tree) + } + } catch (error) { + throw new ExcludeVaultContentError( + `Version comparison failed: ${error.message}`, + tree, + ) + } + }) + return tree + } +} + +const getTfeSemver = (version) => { + // Handle version strings like "1.20.x" by converting to "1.20.0" + const normalized = version.replace(/\.x$/, '.0') + return new SemVer(normalized) +} + +const getComparisonFn = (operator, document) => { + switch (operator) { + case '<=': + return (a, b) => { + return lte(a, b) + } + case '>=': + return (a, b) => { + return gte(a, b) + } + case '<': + return (a, b) => { + return lt(a, b) + } + case '>': + return (a, b) => { + return gt(a, b) + } + case '=': + return (a, b) => { + return eq(a, b) + } + default: + throw new ExcludeVaultContentError( + 'Invalid comparator: ' + operator, + document, + ) + } +} diff --git a/scripts/prebuild/mdx-transforms/exclude-vault-content/index.test.mjs b/scripts/prebuild/mdx-transforms/exclude-vault-content/index.test.mjs new file mode 100644 index 0000000000..4d45fff19e --- /dev/null +++ b/scripts/prebuild/mdx-transforms/exclude-vault-content/index.test.mjs @@ -0,0 +1,271 @@ +/** + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: BUSL-1.1 + */ + +import { describe, it, expect, vi } from 'vitest' +import { transformExcludeVaultContent } from './index.mjs' +import remark from 'remark' +import remarkMdx from 'remark-mdx' + +// Mock for testing custom directive products +vi.mock('../shared.mjs', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + DIRECTIVE_PRODUCTS: ['Vault', 'TFC', 'TFEnterprise'], // Default for most tests + } +}) + +const runTransform = async (markdown, version, filePath) => { + const processor = await remark() + .use(remarkMdx) + .use(transformExcludeVaultContent, { + filePath, + version, + }) + .process(markdown) + return processor.contents +} + +const vaultVersion = '1.20.x' +const filePath = 'vault/some-file.md' + +describe('transformExcludeVaultContent', () => { + it('should remove content when version condition is not met', async () => { + const markdown = ` + +This content should be removed. + +This content should stay. +` + const result = await runTransform(markdown, vaultVersion, filePath) + + expect(result).toBe('This content should stay.\n') + }) + + it('should throw an error for mismatched block names', async () => { + const mockConsole = vi.spyOn(console, 'error').mockImplementation(() => {}) + const markdown = ` + +This content should be removed. + +` + await expect(async () => { + return await runTransform(markdown, vaultVersion, filePath) + }).rejects.toThrow('Mismatched block names') + expect(mockConsole).toHaveBeenCalledOnce() + }) + + it('should throw an error for unexpected END block', async () => { + const markdown = ` + +` + await expect(async () => { + return await runTransform(markdown, vaultVersion, filePath) + }).rejects.toThrow('Unexpected END block') + }) + + it('should throw an error for unexpected BEGIN block', async () => { + const markdown = ` + + +` + await expect(async () => { + return await runTransform(markdown, vaultVersion, filePath) + }).rejects.toThrow('Unexpected BEGIN block') + }) + + it('should throw an error if no block could be parsed from BEGIN comment', async () => { + const markdown = ` + +This content should be removed. + +` + await expect(async () => { + return await runTransform(markdown, vaultVersion, filePath) + }).rejects.toThrow('No block could be parsed from BEGIN comment') + }) + + it('should throw an error if no block could be parsed from END comment', async () => { + const markdown = ` + +This content should be removed. + +` + await expect(async () => { + return await runTransform(markdown, vaultVersion, filePath) + }).rejects.toThrow('No block could be parsed from END comment') + }) + + it('should throw error for names not in directiveProducts array', async () => { + const markdown = ` + +This content should throw an error + +Other content. + ` + await expect(async () => { + return await runTransform(markdown, vaultVersion, filePath) + }).rejects.toThrow( + 'Directive block INVALID:>=v1.21.x could not be parsed between lines 2 and 4', + ) + }) + + it('should keep content when version condition is met', async () => { + const markdown = ` + +This content should stay. + +Other content. +` + const result = await runTransform(markdown, vaultVersion, filePath) + + expect(result.trim()).toBe(` + +This content should stay. + + + +Other content.`) + }) + + it('should handle equality comparisons', async () => { + const equalVersion = '1.20.x' + const markdown = ` + +This content should stay. + +` + const result = await runTransform(markdown, equalVersion, filePath) + + expect(result.trim()).toBe(` + +This content should stay. + +`) + }) + + it('should handle less than comparisons', async () => { + const markdown = ` + +This content should be removed. + +` + const result = await runTransform(markdown, vaultVersion, filePath) + + expect(result.trim()).toBe('') + }) + + it('should throw an error for invalid version format', async () => { + const markdown = ` + +This content should throw an error. + +` + + await expect(async () => { + return await runTransform(markdown, vaultVersion, filePath) + }).rejects.toThrow( + 'Invalid version format in directive: Vault:>=v1.invalid. Expected format: vX.Y.x', + ) + }) + + it('should throw an error for invalid comparator', async () => { + const markdown = ` + +This content should throw an error. + +` + + await expect(async () => { + return await runTransform(markdown, vaultVersion, filePath) + }).rejects.toThrow(/Invalid directive format: Vault:!v1.20.x/) + }) + + it('should handle multiple version blocks correctly', async () => { + const markdown = ` + +This should be removed. + + +This should stay. + +Final content. +` + const result = await runTransform(markdown, vaultVersion, filePath) + + expect(result.trim()).toBe(` + +This should stay. + + + +Final content.`) + }) + + it('should ignore Terraform product directives and leave them untouched', async () => { + const markdown = ` + +This TFC content should be ignored by Vault transform. + + +This TFEnterprise content should also be ignored. + + +This Vault content should be removed. + +Regular content that stays. +` + const result = await runTransform(markdown, vaultVersion, filePath) + + expect(result.trim()).toBe(` + +This TFC content should be ignored by Vault transform. + + + + + +This TFEnterprise content should also be ignored. + + + +Regular content that stays.`) + }) + + it('should throw an error for directives with products not in directiveProducts array', async () => { + // Override DIRECTIVE_PRODUCTS for this test only + vi.doMock('../shared.mjs', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + DIRECTIVE_PRODUCTS: ['Vault', 'TFC'], // CONSUL not included + } + }) + + // Need to re-import the transform after mocking + const { transformExcludeVaultContent: mockTransform } = await import( + './index.mjs' + ) + + const markdown = ` + +This should cause an error - CONSUL not in directiveProducts. + +` + const customRunTransform = async (markdown, version, filePath) => { + const processor = await remark() + .use(remarkMdx) + .use(mockTransform, { filePath, version }) + .process(markdown) + return processor.contents + } + + await expect(async () => { + return await customRunTransform(markdown, vaultVersion, filePath) + }).rejects.toThrow( + 'Directive block CONSUL:>=v1.15.x could not be parsed between lines 2 and 4', + ) + }) +}) diff --git a/scripts/prebuild/prebuild-arm-linux-binary.gz b/scripts/prebuild/prebuild-arm-linux-binary.gz index c68a65e594..217534ad76 100755 Binary files a/scripts/prebuild/prebuild-arm-linux-binary.gz and b/scripts/prebuild/prebuild-arm-linux-binary.gz differ diff --git a/scripts/prebuild/prebuild-arm-mac-binary.gz b/scripts/prebuild/prebuild-arm-mac-binary.gz index 132d354846..6bcdd8a8cf 100755 Binary files a/scripts/prebuild/prebuild-arm-mac-binary.gz and b/scripts/prebuild/prebuild-arm-mac-binary.gz differ diff --git a/scripts/prebuild/prebuild-x64-linux-binary.gz b/scripts/prebuild/prebuild-x64-linux-binary.gz index eda8f30f87..be22d2de64 100755 Binary files a/scripts/prebuild/prebuild-x64-linux-binary.gz and b/scripts/prebuild/prebuild-x64-linux-binary.gz differ