From 8f61f62a564059591b55e7cd16c6f29b4adca0bd Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 8 Jan 2025 20:13:22 +0100 Subject: [PATCH 1/2] Refactor file processing and improve exception handling Extracted file processing logic into a dedicated method for cleaner code organization. Enhanced exception handling with a file processing threshold to fail gracefully after multiple errors. Improved logging to provide better progress tracking. --- .../Diagnostics/DiagnosticsChannel.cs | 1 + .../DocumentationGenerator.cs | 62 ++++++++++++------- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs b/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs index 8d9b3e46a..69b3f5f16 100644 --- a/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs +++ b/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs @@ -154,6 +154,7 @@ public void EmitError(string file, string message, Exception? e = null) }; Channel.Write(d); } + public void EmitWarning(string file, string message) { var d = new Diagnostic diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index 24c029c47..2a4cbf1df 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -85,33 +85,28 @@ public async Task GenerateAll(Cancel ctx) _logger.LogInformation("Resolved tree"); - var handledItems = 0; - + var processedFileCount = 0; + var exceptionCount = 0; _ = Context.Collector.StartAsync(ctx); - await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) => { - if (!Context.Force) + var processedFiles = Interlocked.Increment(ref processedFileCount); + try { - if (offendingFiles.Contains(file.SourceFile.FullName)) - _logger.LogInformation($"Re-evaluating {file.SourceFile.FullName}"); - else if (file.SourceFile.LastWriteTimeUtc <= outputSeenChanges) - return; + await ProcessFile(offendingFiles, file, outputSeenChanges, token); } - - _logger.LogTrace($"{file.SourceFile.FullName}"); - var item = Interlocked.Increment(ref handledItems); - var outputFile = OutputFile(file.RelativePath); - if (file is MarkdownFile markdown) - await HtmlWriter.WriteAsync(outputFile, markdown, token); - else + catch (Exception e) { - if (outputFile.Directory is { Exists: false }) - outputFile.Directory.Create(); - await CopyFileFsAware(file, outputFile, ctx); + var currentCount = Interlocked.Increment(ref exceptionCount); + // this is not the main error logging mechanism + // if we hit this from too many files fail hard + if (currentCount <= 25) + Context.Collector.EmitError(file.RelativePath, "Uncaught exception while processing file", e); + else throw; } - if (item % 1_000 == 0) - _logger.LogInformation($"Handled {handledItems} files"); + + if (processedFiles % 1_000 == 0) + _logger.LogInformation($"Handled {processedFiles} files"); }); var embeddedStaticFiles = Assembly.GetExecutingAssembly() @@ -140,13 +135,34 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) => await GenerateLinkReference(ctx); await Context.Collector.StopAsync(ctx); + } + + private async Task ProcessFile(HashSet offendingFiles, DocumentationFile file, DateTimeOffset outputSeenChanges, CancellationToken token) + { + if (!Context.Force) + { + if (offendingFiles.Contains(file.SourceFile.FullName)) + _logger.LogInformation($"Re-evaluating {file.SourceFile.FullName}"); + else if (file.SourceFile.LastWriteTimeUtc <= outputSeenChanges) + return; + } - IFileInfo OutputFile(string relativePath) + _logger.LogTrace($"{file.SourceFile.FullName}"); + var outputFile = OutputFile(file.RelativePath); + if (file is MarkdownFile markdown) + await HtmlWriter.WriteAsync(outputFile, markdown, token); + else { - var outputFile = _writeFileSystem.FileInfo.New(Path.Combine(DocumentationSet.OutputPath.FullName, relativePath)); - return outputFile; + if (outputFile.Directory is { Exists: false }) + outputFile.Directory.Create(); + await CopyFileFsAware(file, outputFile, token); } + } + private IFileInfo OutputFile(string relativePath) + { + var outputFile = _writeFileSystem.FileInfo.New(Path.Combine(DocumentationSet.OutputPath.FullName, relativePath)); + return outputFile; } private bool CompilationNotNeeded(GenerationState? generationState, out HashSet offendingFiles, From cb18b2c6906f003046769a3c1f4b71cffaed4e1f Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 8 Jan 2025 20:19:49 +0100 Subject: [PATCH 2/2] dotnet format --- src/Elastic.Markdown/DocumentationGenerator.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index 2a4cbf1df..8086e56dd 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -102,7 +102,8 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) => // if we hit this from too many files fail hard if (currentCount <= 25) Context.Collector.EmitError(file.RelativePath, "Uncaught exception while processing file", e); - else throw; + else + throw; } if (processedFiles % 1_000 == 0)