Add the notion of documentation file exporters.

.github/workflows/preview-build.yml

@@ -16,6 +16,10 @@ on:
         description: 'Treat warnings as errors'
         type: string
         default: 'true'
+      metadata-only:
+        description: 'Only generate documentation metadata files'
+        required: false
+        default: 'false'
       continue-on-error:
         description: 'Do not fail to publish if build fails'
         type: string

action.yml

@@ -12,6 +12,9 @@ inputs:
   strict:
     description: 'Treat warnings as errors'
     required: false
+  metadata-only:
+    description: 'Only generate documentation metadata files'
+    required: false
 outputs:
   landing-page-path:
     description: 'Path to the landing page of the documentation'

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -6,6 +6,7 @@
 using System.Reflection;
 using System.Text.Json;
 using Elastic.Markdown.CrossLinks;
+using Elastic.Markdown.Exporters;
 using Elastic.Markdown.IO;
 using Elastic.Markdown.IO.State;
 using Elastic.Markdown.Slices;
@@ -21,10 +22,9 @@ public interface IConversionCollector
 
 public class DocumentationGenerator
 {
-	private readonly IConversionCollector? _conversionCollector;
-	private readonly IFileSystem _readFileSystem;
 	private readonly ILogger _logger;
 	private readonly IFileSystem _writeFileSystem;
+	private readonly IDocumentationFileExporter _documentationFileExporter;
 	private HtmlWriter HtmlWriter { get; }
 
 	public DocumentationSet DocumentationSet { get; }
@@ -34,18 +34,20 @@ public class DocumentationGenerator
 	public DocumentationGenerator(
 		DocumentationSet docSet,
 		ILoggerFactory logger,
+		IDocumentationFileExporter? documentationExporter = null,
 		IConversionCollector? conversionCollector = null
 	)
 	{
-		_conversionCollector = conversionCollector;
-		_readFileSystem = docSet.Build.ReadFileSystem;
 		_writeFileSystem = docSet.Build.WriteFileSystem;
 		_logger = logger.CreateLogger(nameof(DocumentationGenerator));
 
 		DocumentationSet = docSet;
 		Context = docSet.Build;
 		Resolver = docSet.LinkResolver;
 		HtmlWriter = new HtmlWriter(DocumentationSet, _writeFileSystem);
+		_documentationFileExporter =
+			documentationExporter
+			?? new DocumentationFileExporter(docSet.Build.ReadFileSystem, _writeFileSystem, HtmlWriter, conversionCollector);
 
 		_logger.LogInformation("Created documentation set for: {DocumentationSetName}", DocumentationSet.Name);
 		_logger.LogInformation("Source directory: {SourcePath} Exists: {SourcePathExists}", docSet.SourceDirectory, docSet.SourceDirectory.Exists);
@@ -62,7 +64,6 @@ public DocumentationGenerator(
 		return JsonSerializer.Deserialize(contents, SourceGenerationContext.Default.GenerationState);
 	}
 
-
 	public async Task ResolveDirectoryTree(Cancel ctx)
 	{
 		_logger.LogInformation("Resolving tree");
@@ -149,10 +150,7 @@ private async Task ExtractEmbeddedStaticResources(Cancel ctx)
 			var path = a.Replace("Elastic.Markdown.", "").Replace("_static.", "_static/");
 
 			var outputFile = OutputFile(path);
-			if (outputFile.Directory is { Exists: false })
-				outputFile.Directory.Create();
-			await using var stream = outputFile.OpenWrite();
-			await resourceStream.CopyToAsync(stream, ctx);
+			await _documentationFileExporter.CopyEmbeddedResource(outputFile, resourceStream, ctx);
 			_logger.LogDebug("Copied static embedded resource {Path}", path);
 		}
 	}
@@ -169,14 +167,7 @@ private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile
 
 		_logger.LogTrace("--> {FileFullPath}", file.SourceFile.FullName);
 		var outputFile = OutputFile(file.RelativePath);
-		if (file is MarkdownFile markdown)
-			await HtmlWriter.WriteAsync(outputFile, markdown, _conversionCollector, token);
-		else
-		{
-			if (outputFile.Directory is { Exists: false })
-				outputFile.Directory.Create();
-			await CopyFileFsAware(file, outputFile, token);
-		}
+		await _documentationFileExporter.ProcessFile(file, outputFile, token);
 	}
 
 	private IFileInfo OutputFile(string relativePath)
@@ -200,7 +191,8 @@ private bool CompilationNotNeeded(GenerationState? generationState, out HashSet<
 
 		if (Context.Git != generationState.Git)
 		{
-			_logger.LogInformation("Full compilation: current git context: {CurrentGitContext} differs from previous git context: {PreviousGitContext}", Context.Git, generationState.Git);
+			_logger.LogInformation("Full compilation: current git context: {CurrentGitContext} differs from previous git context: {PreviousGitContext}",
+				Context.Git, generationState.Git);
 			return false;
 		}
 
@@ -213,8 +205,10 @@ private bool CompilationNotNeeded(GenerationState? generationState, out HashSet<
 			_logger.LogInformation("Incremental compilation. since: {LastSeenChanges}", generationState.LastSeenChanges);
 		else if (DocumentationSet.LastWrite <= outputSeenChanges)
 		{
-			_logger.LogInformation("No compilation: no changes since last observed: {LastSeenChanges}. " +
-								   "Pass --force to force a full regeneration", generationState.LastSeenChanges);
+			_logger.LogInformation(
+				"No compilation: no changes since last observed: {LastSeenChanges}. " +
+				"Pass --force to force a full regeneration", generationState.LastSeenChanges
+			);
 			return true;
 		}
 
@@ -239,25 +233,13 @@ private async Task GenerateDocumentationState(Cancel ctx)
 		{
 			LastSeenChanges = DocumentationSet.LastWrite,
 			InvalidFiles = badFiles,
-			Git = Context.Git
+			Git = Context.Git,
+			Exporter = _documentationFileExporter.Name
 		};
 		var bytes = JsonSerializer.SerializeToUtf8Bytes(state, SourceGenerationContext.Default.GenerationState);
 		await DocumentationSet.OutputDirectory.FileSystem.File.WriteAllBytesAsync(stateFile.FullName, bytes, ctx);
 	}
 
-	private async Task CopyFileFsAware(DocumentationFile file, IFileInfo outputFile, Cancel ctx)
-	{
-		// fast path, normal case.
-		if (_readFileSystem == _writeFileSystem)
-			_readFileSystem.File.Copy(file.SourceFile.FullName, outputFile.FullName, true);
-		//slower when we are mocking the write filesystem
-		else
-		{
-			var bytes = await file.SourceFile.FileSystem.File.ReadAllBytesAsync(file.SourceFile.FullName, ctx);
-			await outputFile.FileSystem.File.WriteAllBytesAsync(outputFile.FullName, bytes, ctx);
-		}
-	}
-
 	public async Task<string?> RenderLayout(MarkdownFile markdown, Cancel ctx)
 	{
 		await DocumentationSet.Tree.Resolve(ctx);

src/Elastic.Markdown/Exporters/DocumentationFileExporter.cs

@@ -0,0 +1,68 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.IO.Abstractions;
+using Elastic.Markdown.IO;
+using Elastic.Markdown.Slices;
+
+namespace Elastic.Markdown.Exporters;
+
+public interface IDocumentationFileExporter
+{
+	/// Used in documentation state to ensure we break the build cache if a different exporter is chosen
+	string Name { get; }
+
+	Task ProcessFile(DocumentationFile file, IFileInfo outputFile, Cancel token);
+
+	Task CopyEmbeddedResource(IFileInfo outputFile, Stream resourceStream, Cancel ctx);
+}
+
+public abstract class DocumentationFileExporterBase(IFileSystem readFileSystem, IFileSystem writeFileSystem) : IDocumentationFileExporter
+{
+	public abstract string Name { get; }
+	public abstract Task ProcessFile(DocumentationFile file, IFileInfo outputFile, Cancel token);
+
+	protected async Task CopyFileFsAware(DocumentationFile file, IFileInfo outputFile, Cancel ctx)
+	{
+		// fast path, normal case.
+		if (readFileSystem == writeFileSystem)
+			readFileSystem.File.Copy(file.SourceFile.FullName, outputFile.FullName, true);
+		//slower when we are mocking the write filesystem
+		else
+		{
+			var bytes = await file.SourceFile.FileSystem.File.ReadAllBytesAsync(file.SourceFile.FullName, ctx);
+			await outputFile.FileSystem.File.WriteAllBytesAsync(outputFile.FullName, bytes, ctx);
+		}
+	}
+
+	public async Task CopyEmbeddedResource(IFileInfo outputFile, Stream resourceStream, Cancel ctx)
+	{
+		if (outputFile.Directory is { Exists: false })
+			outputFile.Directory.Create();
+		await using var stream = outputFile.OpenWrite();
+		await resourceStream.CopyToAsync(stream, ctx);
+	}
+}
+
+public class DocumentationFileExporter(
+	IFileSystem readFileSystem,
+	IFileSystem writeFileSystem,
+	HtmlWriter htmlWriter,
+	IConversionCollector? conversionCollector
+) : DocumentationFileExporterBase(readFileSystem, writeFileSystem)
+{
+	public override string Name { get; } = nameof(DocumentationFileExporter);
+
+	public override async Task ProcessFile(DocumentationFile file, IFileInfo outputFile, Cancel token)
+	{
+		if (file is MarkdownFile markdown)
+			await htmlWriter.WriteAsync(outputFile, markdown, conversionCollector, token);
+		else
+		{
+			if (outputFile.Directory is { Exists: false })
+				outputFile.Directory.Create();
+			await CopyFileFsAware(file, outputFile, token);
+		}
+	}
+}

src/Elastic.Markdown/Exporters/NoopDocumentationFileExporter.cs

@@ -0,0 +1,15 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.IO.Abstractions;
+using Elastic.Markdown.IO;
+
+namespace Elastic.Markdown.Exporters;
+
+public class NoopDocumentationFileExporter : IDocumentationFileExporter
+{
+	public string Name { get; } = nameof(NoopDocumentationFileExporter);
+	public Task ProcessFile(DocumentationFile file, IFileInfo outputFile, Cancel token) => Task.CompletedTask;
+	public Task CopyEmbeddedResource(IFileInfo outputFile, Stream resourceStream, Cancel ctx) => Task.CompletedTask;
+}

src/Elastic.Markdown/IO/State/GenerationState.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information
 
 using System.Text.Json.Serialization;
+using Elastic.Markdown.Exporters;
 using Elastic.Markdown.IO.Discovery;
 
 namespace Elastic.Markdown.IO.State;
@@ -15,6 +16,9 @@ public record GenerationState
 	[JsonPropertyName("invalid_files")]
 	public required string[] InvalidFiles { get; init; } = [];
 
+	[JsonPropertyName("exporter")]
+	public string Exporter { get; init; } = nameof(DocumentationFileExporter);
+
 	[JsonPropertyName("git")]
 	public required GitCheckoutInformation Git { get; init; }
 }

src/docs-builder/Cli/Commands.cs

@@ -10,6 +10,7 @@
 using Elastic.Documentation.Tooling.Diagnostics.Console;
 using Elastic.Documentation.Tooling.Filters;
 using Elastic.Markdown;
+using Elastic.Markdown.Exporters;
 using Elastic.Markdown.IO;
 using Elastic.Markdown.Refactor;
 using Microsoft.Extensions.Logging;
@@ -56,6 +57,7 @@ public async Task Serve(string? path = null, int port = 3000, Cancel ctx = defau
 	/// <param name="force"> Force a full rebuild of the destination folder</param>
 	/// <param name="strict"> Treat warnings as errors and fail the build on warnings</param>
 	/// <param name="allowIndexing"> Allow indexing and following of html files</param>
+	/// <param name="metadataOnly"> Only emit documentation metadata to output</param>
 	/// <param name="ctx"></param>
 	[Command("generate")]
 	[ConsoleAppFilter<StopwatchFilter>]
@@ -68,6 +70,7 @@ public async Task<int> Generate(
 		bool? force = null,
 		bool? strict = null,
 		bool? allowIndexing = null,
+		bool? metadataOnly = null,
 		Cancel ctx = default
 	)
 	{
@@ -107,7 +110,12 @@ public async Task<int> Generate(
 		if (runningOnCi)
 			await githubActionsService.SetOutputAsync("skip", "false");
 		var set = new DocumentationSet(context, logger);
-		var generator = new DocumentationGenerator(set, logger);
+
+		if (bool.TryParse(githubActionsService.GetInput("metadata-only"), out var metaValue) && metaValue)
+			metadataOnly ??= metaValue;
+		var exporter = metadataOnly.HasValue && metadataOnly.Value ? new NoopDocumentationFileExporter() : null;
+
+		var generator = new DocumentationGenerator(set, logger, exporter);
 		await generator.GenerateAll(ctx);
 		if (runningOnCi)
 			await githubActionsService.SetOutputAsync("landing-page-path", set.MarkdownFiles.First().Value.Url);
@@ -128,6 +136,7 @@ public async Task<int> Generate(
 	/// <param name="force"> Force a full rebuild of the destination folder</param>
 	/// <param name="strict"> Treat warnings as errors and fail the build on warnings</param>
 	/// <param name="allowIndexing"> Allow indexing and following of html files</param>
+	/// <param name="metadataOnly"> Only emit documentation metadata to output</param>
 	/// <param name="ctx"></param>
 	[Command("")]
 	[ConsoleAppFilter<StopwatchFilter>]
@@ -140,9 +149,10 @@ public async Task<int> GenerateDefault(
 		bool? force = null,
 		bool? strict = null,
 		bool? allowIndexing = null,
+		bool? metadataOnly = null,
 		Cancel ctx = default
 	) =>
-		await Generate(path, output, pathPrefix, force, strict, allowIndexing, ctx);
+		await Generate(path, output, pathPrefix, force, strict, allowIndexing, metadataOnly, ctx);
 
 
 	/// <summary>

tests/authoring/Framework/Setup.fs

@@ -102,7 +102,7 @@ type Setup =
         let conversionCollector = TestConversionCollector()
         let linkResolver = TestCrossLinkResolver(context.Configuration)
         let set = DocumentationSet(context, logger, linkResolver);
-        let generator = DocumentationGenerator(set, logger, conversionCollector)
+        let generator = DocumentationGenerator(set, logger, null, conversionCollector)
 
         let context = {
             Collector = collector