From 60309b3541b13e0b2d82290c083b6027b2da8fcc Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 23 Sep 2025 20:37:29 +0200 Subject: [PATCH 1/6] Add specialized index commands --- .../DocumentationEndpoints.cs | 40 ++++- src/Elastic.Documentation/Exporter.cs | 2 +- .../ElasticsearchMarkdownExporter.cs | 51 ++++--- .../Exporters/ExporterExtensions.cs | 12 +- .../Building/AssemblerBuilder.cs | 3 +- .../Indexing/AssemblerIndexService.cs | 138 ++++++++++++++++++ .../IsolatedBuildService.cs | 2 +- .../IsolatedIndexService.cs | 138 ++++++++++++++++++ .../Arguments/ExportOption.cs | 1 - .../Assembler/AssemblerIndexCommand.cs | 119 +++++++++++++++ .../Commands/AssemblerIndexCommand.cs | 119 +++++++++++++++ src/tooling/docs-builder/Program.cs | 3 +- 12 files changed, 589 insertions(+), 39 deletions(-) create mode 100644 src/services/Elastic.Documentation.Assembler/Indexing/AssemblerIndexService.cs create mode 100644 src/services/Elastic.Documentation.Isolated/IsolatedIndexService.cs create mode 100644 src/tooling/docs-builder/Commands/Assembler/AssemblerIndexCommand.cs create mode 100644 src/tooling/docs-builder/Commands/AssemblerIndexCommand.cs diff --git a/src/Elastic.Documentation.Configuration/DocumentationEndpoints.cs b/src/Elastic.Documentation.Configuration/DocumentationEndpoints.cs index 3f5c6c3fd..21afc6ac4 100644 --- a/src/Elastic.Documentation.Configuration/DocumentationEndpoints.cs +++ b/src/Elastic.Documentation.Configuration/DocumentationEndpoints.cs @@ -2,19 +2,45 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.Security.Cryptography.X509Certificates; + namespace Elastic.Documentation.Configuration; -public record DocumentationEndpoints +public class DocumentationEndpoints { public required ElasticsearchEndpoint Elasticsearch { get; init; } } -public record ElasticsearchEndpoint +public class ElasticsearchEndpoint { - public static ElasticsearchEndpoint Default { get; } = new ElasticsearchEndpoint { Uri = new Uri("https://localhost:9200") }; + public static ElasticsearchEndpoint Default { get; } = new() { Uri = new Uri("https://localhost:9200") }; + + public required Uri Uri { get; set; } + public string? Username { get; set; } + public string? Password { get; set; } + public string? ApiKey { get; set; } + + // inference options + public int SearchNumThreads { get; set; } = 8; + public int IndexNumThreads { get; set; } = 8; + + // index options + public string IndexNamePrefix { get; set; } = "semantic-docs"; + + // channel buffer options + public int BufferSize { get; set; } = 100; + public int MaxRetries { get; set; } = 3; + + + // connection options + public bool DebugMode { get; set; } + public string? CertificateFingerprint { get; set; } + public string? ProxyAddress { get; set; } + public string? ProxyPassword { get; set; } + public string? ProxyUsername { get; set; } - public required Uri Uri { get; init; } - public string? Username { get; init; } - public string? Password { get; init; } - public string? ApiKey { get; init; } + public bool DisableSslVerification { get; set; } + public X509Certificate? Certificate { get; set; } + public bool CertificateIsNotRoot { get; set; } + public int? BootstrapTimeout { get; set; } } diff --git a/src/Elastic.Documentation/Exporter.cs b/src/Elastic.Documentation/Exporter.cs index 199301994..6af9ed524 100644 --- a/src/Elastic.Documentation/Exporter.cs +++ b/src/Elastic.Documentation/Exporter.cs @@ -11,7 +11,7 @@ public enum Exporter Html, LLMText, Elasticsearch, - SemanticElasticsearch, + ElasticsearchNoSemantic, Configuration, DocumentationState, LinkMetadata, diff --git a/src/Elastic.Markdown/Exporters/ElasticsearchMarkdownExporter.cs b/src/Elastic.Markdown/Exporters/ElasticsearchMarkdownExporter.cs index bda7c8744..a72761529 100644 --- a/src/Elastic.Markdown/Exporters/ElasticsearchMarkdownExporter.cs +++ b/src/Elastic.Markdown/Exporters/ElasticsearchMarkdownExporter.cs @@ -5,7 +5,6 @@ using System.IO.Abstractions; using Elastic.Channels; using Elastic.Documentation.Configuration; -using Elastic.Documentation.Configuration.Assembler; using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Search; using Elastic.Documentation.Serialization; @@ -20,7 +19,7 @@ namespace Elastic.Markdown.Exporters; -public class ElasticsearchMarkdownExporter(ILoggerFactory logFactory, IDiagnosticsCollector collector, DocumentationEndpoints endpoints) +public class ElasticsearchMarkdownExporter(ILoggerFactory logFactory, IDiagnosticsCollector collector, string indexNamespace, DocumentationEndpoints endpoints) : ElasticsearchMarkdownExporterBase, CatalogIndexChannel> (logFactory, collector, endpoints) { @@ -28,15 +27,15 @@ public class ElasticsearchMarkdownExporter(ILoggerFactory logFactory, IDiagnosti protected override CatalogIndexChannelOptions NewOptions(DistributedTransport transport) => new(transport) { GetMapping = () => CreateMapping(null), - IndexFormat = "documentation{0:yyyy.MM.dd.HHmmss}", - ActiveSearchAlias = "documentation" + IndexFormat = $"{Endpoint.IndexNamePrefix.ToLowerInvariant()}-{indexNamespace.ToLowerInvariant()}-{{0:yyyy.MM.dd.HHmmss}}", + ActiveSearchAlias = $"{Endpoint.IndexNamePrefix}-{indexNamespace.ToLowerInvariant()}", }; /// protected override CatalogIndexChannel NewChannel(CatalogIndexChannelOptions options) => new(options); } -public class ElasticsearchMarkdownSemanticExporter(PublishEnvironment environment, ILoggerFactory logFactory, IDiagnosticsCollector collector, DocumentationEndpoints endpoints) +public class ElasticsearchMarkdownSemanticExporter(ILoggerFactory logFactory, IDiagnosticsCollector collector, string indexNamespace, DocumentationEndpoints endpoints) : ElasticsearchMarkdownExporterBase, SemanticIndexChannel> (logFactory, collector, endpoints) { @@ -45,20 +44,23 @@ public class ElasticsearchMarkdownSemanticExporter(PublishEnvironment environmen { GetMapping = (inferenceId, _) => CreateMapping(inferenceId), GetMappingSettings = (_, _) => CreateMappingSetting(), - IndexFormat = $"semantic-docs-{environment.Name}-{{0:yyyy.MM.dd.HHmmss}}", - ActiveSearchAlias = $"semantic-docs-{environment.Name}", - IndexNumThreads = IndexNumThreads, - InferenceCreateTimeout = TimeSpan.FromMinutes(4) + IndexFormat = $"{Endpoint.IndexNamePrefix.ToLowerInvariant()}-{indexNamespace.ToLowerInvariant()}-{{0:yyyy.MM.dd.HHmmss}}", + ActiveSearchAlias = $"{Endpoint.IndexNamePrefix}-{indexNamespace.ToLowerInvariant()}", + IndexNumThreads = Endpoint.IndexNumThreads, + SearchNumThreads = Endpoint.SearchNumThreads, + InferenceCreateTimeout = TimeSpan.FromMinutes(Endpoint.BootstrapTimeout ?? 4) }; /// protected override SemanticIndexChannel NewChannel(SemanticIndexChannelOptions options) => new(options); } + public abstract class ElasticsearchMarkdownExporterBase( ILoggerFactory logFactory, IDiagnosticsCollector collector, - DocumentationEndpoints endpoints) + DocumentationEndpoints endpoints +) : IMarkdownExporter, IDisposable where TChannelOptions : CatalogIndexChannelOptionsBase where TChannel : CatalogIndexChannel @@ -69,7 +71,7 @@ public abstract class ElasticsearchMarkdownExporterBase 8; + protected ElasticsearchEndpoint Endpoint { get; } = endpoints.Elasticsearch; protected static string CreateMappingSetting() => // language=json @@ -97,7 +99,6 @@ protected static string CreateMappingSetting() => """; protected static string CreateMapping(string? inferenceId) => - // langugage=json $$""" { "properties": { @@ -131,7 +132,6 @@ protected static string CreateMapping(string? inferenceId) => """; private static string AbstractMapping() => - // langugage=json """ , "abstract": { "type": "text" @@ -139,7 +139,6 @@ private static string AbstractMapping() => """; private static string InferenceMapping(string inferenceId) => - // langugage=json $""" "type": "semantic_text", "inference_id": "{inferenceId}" @@ -159,12 +158,26 @@ public async ValueTask StartAsync(Cancel ctx = default) return; var es = endpoints.Elasticsearch; + var configuration = new ElasticsearchConfiguration(es.Uri) { Authentication = es.ApiKey is { } apiKey ? new ApiKey(apiKey) - : es.Username is { } username && es.Password is { } password + : es is { Username: { } username, Password: { } password } ? new BasicAuthentication(username, password) + : null, + EnableHttpCompression = true, + DebugMode = Endpoint.DebugMode, + CertificateFingerprint = Endpoint.CertificateFingerprint, + ProxyAddress = Endpoint.ProxyAddress, + ProxyPassword = Endpoint.ProxyPassword, + ProxyUsername = Endpoint.ProxyUsername, + ServerCertificateValidationCallback = Endpoint.DisableSslVerification + ? CertificateValidations.AllowAll + : Endpoint.Certificate is { } cert + ? Endpoint.CertificateIsNotRoot + ? CertificateValidations.AuthorityPartOfChain(cert) + : CertificateValidations.AuthorityIsRoot(cert) : null }; @@ -175,9 +188,9 @@ public async ValueTask StartAsync(Cancel ctx = default) var options = NewOptions(transport); options.BufferOptions = new BufferOptions { - OutboundBufferMaxSize = 100, - ExportMaxConcurrency = IndexNumThreads, - ExportMaxRetries = 3 + OutboundBufferMaxSize = Endpoint.BufferSize, + ExportMaxConcurrency = Endpoint.IndexNumThreads, + ExportMaxRetries = Endpoint.MaxRetries, }; options.SerializerContext = SourceGenerationContext.Default; options.ExportBufferCallback = () => _logger.LogInformation("Exported buffer to Elasticsearch"); @@ -206,7 +219,7 @@ public async ValueTask StopAsync(Cancel ctx = default) _logger.LogInformation("Applying aliases to {Index}", _channel.IndexName); var swapped = await _channel.ApplyAliasesAsync(ctx); if (!swapped) - collector.EmitGlobalError($"{nameof(ElasticsearchMarkdownExporter)} failed to apply aliases to index {_channel.IndexName}"); + collector.EmitGlobalError($"${nameof(ElasticsearchMarkdownExporter)} failed to apply aliases to index {_channel.IndexName}"); } public void Dispose() diff --git a/src/Elastic.Markdown/Exporters/ExporterExtensions.cs b/src/Elastic.Markdown/Exporters/ExporterExtensions.cs index f54234792..c544d887a 100644 --- a/src/Elastic.Markdown/Exporters/ExporterExtensions.cs +++ b/src/Elastic.Markdown/Exporters/ExporterExtensions.cs @@ -15,7 +15,7 @@ public static IReadOnlyCollection CreateMarkdownExporters( this IReadOnlySet exportOptions, ILoggerFactory logFactory, IDocumentationConfigurationContext context, - PublishEnvironment? environment = null + string indexNamespace ) { var markdownExporters = new List(3); @@ -24,13 +24,9 @@ public static IReadOnlyCollection CreateMarkdownExporters( if (exportOptions.Contains(Exporter.Configuration)) markdownExporters.Add(new ConfigurationExporter(logFactory, context.ConfigurationFileProvider, context)); if (exportOptions.Contains(Exporter.Elasticsearch)) - markdownExporters.Add(new ElasticsearchMarkdownExporter(logFactory, context.Collector, context.Endpoints)); - if (exportOptions.Contains(Exporter.SemanticElasticsearch)) - { - if (environment is null) - throw new ArgumentNullException(nameof(environment), "A publish environment is required when using the semantic elasticsearch exporter"); - markdownExporters.Add(new ElasticsearchMarkdownSemanticExporter(environment, logFactory, context.Collector, context.Endpoints)); - } + markdownExporters.Add(new ElasticsearchMarkdownSemanticExporter(logFactory, context.Collector, indexNamespace, context.Endpoints)); + if (exportOptions.Contains(Exporter.ElasticsearchNoSemantic)) + markdownExporters.Add(new ElasticsearchMarkdownExporter(logFactory, context.Collector, indexNamespace, context.Endpoints)); return markdownExporters; } } diff --git a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs index f92c965b4..84c6a3747 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs @@ -13,6 +13,7 @@ using Elastic.Documentation.Serialization; using Elastic.Markdown; using Elastic.Markdown.Exporters; +using Elastic.Markdown.Helpers; using Microsoft.Extensions.Logging; namespace Elastic.Documentation.Assembler.Building; @@ -40,7 +41,7 @@ public async Task BuildAllAsync(PublishEnvironment environment, FrozenDictionary var redirects = new Dictionary(); - var markdownExporters = exportOptions.CreateMarkdownExporters(logFactory, context, environment); + var markdownExporters = exportOptions.CreateMarkdownExporters(logFactory, context, environment.Name); var tasks = markdownExporters.Select(async e => await e.StartAsync(ctx)); await Task.WhenAll(tasks); diff --git a/src/services/Elastic.Documentation.Assembler/Indexing/AssemblerIndexService.cs b/src/services/Elastic.Documentation.Assembler/Indexing/AssemblerIndexService.cs new file mode 100644 index 000000000..0a8b6bf96 --- /dev/null +++ b/src/services/Elastic.Documentation.Assembler/Indexing/AssemblerIndexService.cs @@ -0,0 +1,138 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using System.Security.Cryptography.X509Certificates; +using Actions.Core.Services; +using Elastic.Documentation.Assembler.Building; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.Assembler; +using Elastic.Documentation.Diagnostics; +using Microsoft.Extensions.Logging; +using static Elastic.Documentation.Exporter; + +namespace Elastic.Documentation.Assembler.Indexing; + +public class AssemblerIndexService( + ILoggerFactory logFactory, + AssemblyConfiguration assemblyConfiguration, + IConfigurationContext configurationContext, + ICoreService githubActionsService +) : AssemblerBuildService(logFactory, assemblyConfiguration, configurationContext, githubActionsService) +{ + private readonly IConfigurationContext _configurationContext = configurationContext; + + /// + /// Index documentation to Elasticsearch, calls `docs-builder assembler build --exporters elasticsearch`. Exposes more options + /// + /// + /// + /// Elasticsearch endpoint, alternatively set env DOCUMENTATION_ELASTIC_URL + /// The --environment used to clone ends up being part of the index name + /// Elasticsearch API key, alternatively set env DOCUMENTATION_ELASTIC_APIKEY + /// Elasticsearch username (basic auth), alternatively set env DOCUMENTATION_ELASTIC_USERNAME + /// Elasticsearch password (basic auth), alternatively set env DOCUMENTATION_ELASTIC_PASSWORD + /// Index without semantic fields + /// The number of search threads the inference endpoint should use. Defaults: 8 + /// The number of index threads the inference endpoint should use. Defaults: 8 + /// Timeout in minutes for the inference endpoint creation. Defaults: 4 + /// The prefix for the computed index/alias names. Defaults: semantic-docs + /// The number of documents to send to ES as part of the bulk. Defaults: 100 + /// The number of times failed bulk items should be retried. Defaults: 3 + /// Buffer ES request/responses for better error messages and pass ?pretty to all requests + /// Route requests through a proxy server + /// Proxy server password + /// Proxy server username + /// Disable SSL certificate validation (EXPERT OPTION) + /// Pass a self-signed certificate fingerprint to validate the SSL connection + /// Pass a self-signed certificate to validate the SSL connection + /// If the certificate is not root but only part of the validation chain pass this + /// + /// + public async Task Index(IDiagnosticsCollector collector, + FileSystem fileSystem, + string? endpoint = null, + string? environment = null, + string? apiKey = null, + string? username = null, + string? password = null, + // inference options + bool? noSemantic = null, + int? searchNumThreads = null, + int? indexNumThreads = null, + int? bootstrapTimeout = null, + // index options + string? indexNamePrefix = null, + // channel buffer options + int? bufferSize = null, + int? maxRetries = null, + // connection options + bool? debugMode = null, + string? proxyAddress = null, + string? proxyPassword = null, + string? proxyUsername = null, + bool? disableSslVerification = null, + string? certificateFingerprint = null, + string? certificatePath = null, + bool? certificateNotRoot = null, + Cancel ctx = default + ) + { + var cfg = _configurationContext.Endpoints.Elasticsearch; + if (!string.IsNullOrEmpty(endpoint)) + { + if (!Uri.TryCreate(endpoint, UriKind.Absolute, out var uri)) + collector.EmitGlobalError($"'{endpoint}' is not a valid URI"); + else + cfg.Uri = uri; + } + + if (!string.IsNullOrEmpty(apiKey)) + cfg.ApiKey = apiKey; + if (!string.IsNullOrEmpty(username)) + cfg.Username = username; + if (!string.IsNullOrEmpty(password)) + cfg.Password = password; + + if (searchNumThreads.HasValue) + cfg.SearchNumThreads = searchNumThreads.Value; + if (indexNumThreads.HasValue) + cfg.IndexNumThreads = indexNumThreads.Value; + if (!string.IsNullOrEmpty(indexNamePrefix)) + cfg.IndexNamePrefix = indexNamePrefix; + if (bufferSize.HasValue) + cfg.BufferSize = bufferSize.Value; + if (maxRetries.HasValue) + cfg.MaxRetries = maxRetries.Value; + if (debugMode.HasValue) + cfg.DebugMode = debugMode.Value; + if (!string.IsNullOrEmpty(certificateFingerprint)) + cfg.CertificateFingerprint = certificateFingerprint; + if (!string.IsNullOrEmpty(proxyAddress)) + cfg.ProxyAddress = proxyAddress; + if (!string.IsNullOrEmpty(proxyPassword)) + cfg.ProxyPassword = proxyPassword; + if (!string.IsNullOrEmpty(proxyUsername)) + cfg.ProxyUsername = proxyUsername; + if (disableSslVerification.HasValue) + cfg.DisableSslVerification = disableSslVerification.Value; + if (!string.IsNullOrEmpty(certificatePath)) + { + if (!fileSystem.File.Exists(certificatePath)) + collector.EmitGlobalError($"'{certificatePath}' does not exist"); + var bytes = await fileSystem.File.ReadAllBytesAsync(certificatePath, ctx); + var loader = X509CertificateLoader.LoadCertificate(bytes); + cfg.Certificate = loader; + } + + if (certificateNotRoot.HasValue) + cfg.CertificateIsNotRoot = certificateNotRoot.Value; + if (bootstrapTimeout.HasValue) + cfg.BootstrapTimeout = bootstrapTimeout.Value; + + var exporters = new HashSet { noSemantic.GetValueOrDefault(false) ? ElasticsearchNoSemantic : Elasticsearch }; + + return await BuildAll(collector, strict: false, environment, metadataOnly: true, exporters, fileSystem, ctx); + } +} diff --git a/src/services/Elastic.Documentation.Isolated/IsolatedBuildService.cs b/src/services/Elastic.Documentation.Isolated/IsolatedBuildService.cs index 5582a5290..70d6efa11 100644 --- a/src/services/Elastic.Documentation.Isolated/IsolatedBuildService.cs +++ b/src/services/Elastic.Documentation.Isolated/IsolatedBuildService.cs @@ -112,7 +112,7 @@ public async Task Build( if (runningOnCi) set.ClearOutputDirectory(); - var markdownExporters = exporters.CreateMarkdownExporters(logFactory, context); + var markdownExporters = exporters.CreateMarkdownExporters(logFactory, context, "isolated"); var generator = new DocumentationGenerator(set, logFactory, null, null, markdownExporters.ToArray()); _ = await generator.GenerateAll(ctx); diff --git a/src/services/Elastic.Documentation.Isolated/IsolatedIndexService.cs b/src/services/Elastic.Documentation.Isolated/IsolatedIndexService.cs new file mode 100644 index 000000000..b7ef38baf --- /dev/null +++ b/src/services/Elastic.Documentation.Isolated/IsolatedIndexService.cs @@ -0,0 +1,138 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using System.Security.Cryptography.X509Certificates; +using Actions.Core.Services; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Diagnostics; +using Microsoft.Extensions.Logging; +using static Elastic.Documentation.Exporter; + +namespace Elastic.Documentation.Isolated; + +public class IsolatedIndexService( + ILoggerFactory logFactory, + IConfigurationContext configurationContext, + ICoreService githubActionsService +) : IsolatedBuildService(logFactory, configurationContext, githubActionsService) +{ + private readonly IConfigurationContext _configurationContext = configurationContext; + + /// + /// Index documentation to Elasticsearch, calls `docs-builder assembler build --exporters elasticsearch`. Exposes more options + /// + /// + /// + /// path to the documentation folder, defaults to pwd. + /// Elasticsearch endpoint, alternatively set env DOCUMENTATION_ELASTIC_URL + /// Elasticsearch API key, alternatively set env DOCUMENTATION_ELASTIC_APIKEY + /// Elasticsearch username (basic auth), alternatively set env DOCUMENTATION_ELASTIC_USERNAME + /// Elasticsearch password (basic auth), alternatively set env DOCUMENTATION_ELASTIC_PASSWORD + /// Index without semantic fields + /// The number of search threads the inference endpoint should use. Defaults: 8 + /// The number of index threads the inference endpoint should use. Defaults: 8 + /// Timeout in minutes for the inference endpoint creation. Defaults: 4 + /// The prefix for the computed index/alias names. Defaults: semantic-docs + /// The number of documents to send to ES as part of the bulk. Defaults: 100 + /// The number of times failed bulk items should be retried. Defaults: 3 + /// Buffer ES request/responses for better error messages and pass ?pretty to all requests + /// Route requests through a proxy server + /// Proxy server password + /// Proxy server username + /// Disable SSL certificate validation (EXPERT OPTION) + /// Pass a self-signed certificate fingerprint to validate the SSL connection + /// Pass a self-signed certificate to validate the SSL connection + /// If the certificate is not root but only part of the validation chain pass this + /// + /// + public async Task Index(IDiagnosticsCollector collector, + FileSystem fileSystem, + string? path = null, + string? endpoint = null, + string? apiKey = null, + string? username = null, + string? password = null, + // inference options + bool? noSemantic = null, + int? searchNumThreads = null, + int? indexNumThreads = null, + int? bootstrapTimeout = null, + // index options + string? indexNamePrefix = null, + // channel buffer options + int? bufferSize = null, + int? maxRetries = null, + // connection options + bool? debugMode = null, + string? proxyAddress = null, + string? proxyPassword = null, + string? proxyUsername = null, + bool? disableSslVerification = null, + string? certificateFingerprint = null, + string? certificatePath = null, + bool? certificateNotRoot = null, + Cancel ctx = default + ) + { + var cfg = _configurationContext.Endpoints.Elasticsearch; + if (!string.IsNullOrEmpty(endpoint)) + { + if (!Uri.TryCreate(endpoint, UriKind.Absolute, out var uri)) + collector.EmitGlobalError($"'{endpoint}' is not a valid URI"); + else + cfg.Uri = uri; + } + + if (!string.IsNullOrEmpty(apiKey)) + cfg.ApiKey = apiKey; + if (!string.IsNullOrEmpty(username)) + cfg.Username = username; + if (!string.IsNullOrEmpty(password)) + cfg.Password = password; + + if (searchNumThreads.HasValue) + cfg.SearchNumThreads = searchNumThreads.Value; + if (indexNumThreads.HasValue) + cfg.IndexNumThreads = indexNumThreads.Value; + if (!string.IsNullOrEmpty(indexNamePrefix)) + cfg.IndexNamePrefix = indexNamePrefix; + if (bufferSize.HasValue) + cfg.BufferSize = bufferSize.Value; + if (maxRetries.HasValue) + cfg.MaxRetries = maxRetries.Value; + if (debugMode.HasValue) + cfg.DebugMode = debugMode.Value; + if (!string.IsNullOrEmpty(certificateFingerprint)) + cfg.CertificateFingerprint = certificateFingerprint; + if (!string.IsNullOrEmpty(proxyAddress)) + cfg.ProxyAddress = proxyAddress; + if (!string.IsNullOrEmpty(proxyPassword)) + cfg.ProxyPassword = proxyPassword; + if (!string.IsNullOrEmpty(proxyUsername)) + cfg.ProxyUsername = proxyUsername; + if (disableSslVerification.HasValue) + cfg.DisableSslVerification = disableSslVerification.Value; + if (!string.IsNullOrEmpty(certificatePath)) + { + if (!fileSystem.File.Exists(certificatePath)) + collector.EmitGlobalError($"'{certificatePath}' does not exist"); + var bytes = await fileSystem.File.ReadAllBytesAsync(certificatePath, ctx); + var loader = X509CertificateLoader.LoadCertificate(bytes); + cfg.Certificate = loader; + } + + if (certificateNotRoot.HasValue) + cfg.CertificateIsNotRoot = certificateNotRoot.Value; + if (bootstrapTimeout.HasValue) + cfg.BootstrapTimeout = bootstrapTimeout.Value; + + var exporters = new HashSet { noSemantic.GetValueOrDefault(false) ? ElasticsearchNoSemantic : Elasticsearch }; + + return await Build(collector, fileSystem, + metadataOnly: true, strict: false, path: path, output: null, pathPrefix: null, + force: true, allowIndexing: null, exporters: exporters, canonicalBaseUrl: null, + ctx: ctx); + } +} diff --git a/src/tooling/Elastic.Documentation.Tooling/Arguments/ExportOption.cs b/src/tooling/Elastic.Documentation.Tooling/Arguments/ExportOption.cs index fa706f4c6..936f9bbf6 100644 --- a/src/tooling/Elastic.Documentation.Tooling/Arguments/ExportOption.cs +++ b/src/tooling/Elastic.Documentation.Tooling/Arguments/ExportOption.cs @@ -23,7 +23,6 @@ public static bool TryParse(ReadOnlySpan s, out IReadOnlySet res "llmtext" => LLMText, "es" => Elasticsearch, "elasticsearch" => Elasticsearch, - "semantic" => SemanticElasticsearch, "html" => Html, "config" => Exporter.Configuration, "links" => LinkMetadata, diff --git a/src/tooling/docs-builder/Commands/Assembler/AssemblerIndexCommand.cs b/src/tooling/docs-builder/Commands/Assembler/AssemblerIndexCommand.cs new file mode 100644 index 000000000..cef1335e3 --- /dev/null +++ b/src/tooling/docs-builder/Commands/Assembler/AssemblerIndexCommand.cs @@ -0,0 +1,119 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using Actions.Core.Services; +using ConsoleAppFramework; +using Elastic.Documentation.Assembler.Indexing; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.Assembler; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Services; +using Microsoft.Extensions.Logging; + +namespace Documentation.Builder.Commands.Assembler; + +internal sealed class AssemblerIndexCommand( + ILoggerFactory logFactory, + IDiagnosticsCollector collector, + AssemblyConfiguration configuration, + IConfigurationContext configurationContext, + ICoreService githubActionsService +) +{ + /// + /// Index documentation to Elasticsearch, calls `docs-builder assembler build --exporters elasticsearch`. Exposes more options + /// + /// Elasticsearch endpoint, alternatively set env DOCUMENTATION_ELASTIC_URL + /// The --environment used to clone ends up being part of the index name + /// Elasticsearch API key, alternatively set env DOCUMENTATION_ELASTIC_APIKEY + /// Elasticsearch username (basic auth), alternatively set env DOCUMENTATION_ELASTIC_USERNAME + /// Elasticsearch password (basic auth), alternatively set env DOCUMENTATION_ELASTIC_PASSWORD + /// Index without semantic fields + /// The number of search threads the inference endpoint should use. Defaults: 8 + /// The number of index threads the inference endpoint should use. Defaults: 8 + /// The prefix for the computed index/alias names. Defaults: semantic-docs + /// Timeout in minutes for the inference endpoint creation. Defaults: 4 + /// The number of documents to send to ES as part of the bulk. Defaults: 100 + /// The number of times failed bulk items should be retried. Defaults: 3 + /// Buffer ES request/responses for better error messages and pass ?pretty to all requests + /// Route requests through a proxy server + /// Proxy server password + /// Proxy server username + /// Disable SSL certificate validation (EXPERT OPTION) + /// Pass a self-signed certificate fingerprint to validate the SSL connection + /// Pass a self-signed certificate to validate the SSL connection + /// If the certificate is not root but only part of the validation chain pass this + /// + /// + [Command("")] + public async Task Index( + [Argument] string? endpoint = null, + string? environment = null, + string? apiKey = null, + string? username = null, + string? password = null, + + // inference options + bool? noSemantic = null, + int? searchNumThreads = null, + int? indexNumThreads = null, + int? bootstrapTimeout = null, + + // index options + string? indexNamePrefix = null, + + // channel buffer options + int? bufferSize = null, + int? maxRetries = null, + + // connection options + bool? debugMode = null, + + // proxy options + string? proxyAddress = null, + string? proxyPassword = null, + string? proxyUsername = null, + + // certificate options + bool? disableSslVerification = null, + string? certificateFingerprint = null, + string? certificatePath = null, + bool? certificateNotRoot = null, + Cancel ctx = default + ) + { + await using var serviceInvoker = new ServiceInvoker(collector); + var fs = new FileSystem(); + var service = new AssemblerIndexService(logFactory, configuration, configurationContext, githubActionsService); + var state = (fs, + // endpoint options + endpoint, environment, apiKey, username, password, + // inference options + noSemantic, indexNumThreads, searchNumThreads, bootstrapTimeout, + // channel and connection options + indexNamePrefix, bufferSize, maxRetries, debugMode, + // proxy options + proxyAddress, proxyPassword, proxyUsername, + // certificate options + disableSslVerification, certificateFingerprint, certificatePath, certificateNotRoot + ); + serviceInvoker.AddCommand(service, state, + static async (s, collector, state, ctx) => await s.Index(collector, state.fs, + // endpoint options + state.endpoint, state.environment, state.apiKey, state.username, state.password, + // inference options + state.noSemantic, state.searchNumThreads, state.indexNumThreads, state.bootstrapTimeout, + // channel and connection options + state.indexNamePrefix, state.bufferSize, state.maxRetries, state.debugMode, + // proxy options + state.proxyAddress, state.proxyPassword, state.proxyUsername, + // certificate options + state.disableSslVerification, state.certificateFingerprint, state.certificatePath, state.certificateNotRoot + , ctx) + ); + + return await serviceInvoker.InvokeAsync(ctx); + } +} diff --git a/src/tooling/docs-builder/Commands/AssemblerIndexCommand.cs b/src/tooling/docs-builder/Commands/AssemblerIndexCommand.cs new file mode 100644 index 000000000..e93332cd5 --- /dev/null +++ b/src/tooling/docs-builder/Commands/AssemblerIndexCommand.cs @@ -0,0 +1,119 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using Actions.Core.Services; +using ConsoleAppFramework; +using Elastic.Documentation.Assembler.Indexing; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.Assembler; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Isolated; +using Elastic.Documentation.Services; +using Microsoft.Extensions.Logging; + +namespace Documentation.Builder.Commands; + +internal sealed class IndexCommand( + ILoggerFactory logFactory, + IDiagnosticsCollector collector, + IConfigurationContext configurationContext, + ICoreService githubActionsService +) +{ + /// + /// Index a single documentation set to Elasticsearch, calls `docs-builder --exporters elasticsearch`. Exposes more options + /// + /// Elasticsearch endpoint, alternatively set env DOCUMENTATION_ELASTIC_URL + /// The --environment used to clone ends up being part of the index name + /// Elasticsearch API key, alternatively set env DOCUMENTATION_ELASTIC_APIKEY + /// Elasticsearch username (basic auth), alternatively set env DOCUMENTATION_ELASTIC_USERNAME + /// Elasticsearch password (basic auth), alternatively set env DOCUMENTATION_ELASTIC_PASSWORD + /// Index without semantic fields + /// The number of search threads the inference endpoint should use. Defaults: 8 + /// The number of index threads the inference endpoint should use. Defaults: 8 + /// The prefix for the computed index/alias names. Defaults: semantic-docs + /// Timeout in minutes for the inference endpoint creation. Defaults: 4 + /// The number of documents to send to ES as part of the bulk. Defaults: 100 + /// The number of times failed bulk items should be retried. Defaults: 3 + /// Buffer ES request/responses for better error messages and pass ?pretty to all requests + /// Route requests through a proxy server + /// Proxy server password + /// Proxy server username + /// Disable SSL certificate validation (EXPERT OPTION) + /// Pass a self-signed certificate fingerprint to validate the SSL connection + /// Pass a self-signed certificate to validate the SSL connection + /// If the certificate is not root but only part of the validation chain pass this + /// + /// + [Command("")] + public async Task Index( + [Argument] string? endpoint = null, + string? environment = null, + string? apiKey = null, + string? username = null, + string? password = null, + + // inference options + bool? noSemantic = null, + int? searchNumThreads = null, + int? indexNumThreads = null, + int? bootstrapTimeout = null, + + // index options + string? indexNamePrefix = null, + + // channel buffer options + int? bufferSize = null, + int? maxRetries = null, + + // connection options + bool? debugMode = null, + + // proxy options + string? proxyAddress = null, + string? proxyPassword = null, + string? proxyUsername = null, + + // certificate options + bool? disableSslVerification = null, + string? certificateFingerprint = null, + string? certificatePath = null, + bool? certificateNotRoot = null, + Cancel ctx = default + ) + { + await using var serviceInvoker = new ServiceInvoker(collector); + var fs = new FileSystem(); + var service = new IsolatedIndexService(logFactory, configurationContext, githubActionsService); + var state = (fs, + // endpoint options + endpoint, environment, apiKey, username, password, + // inference options + noSemantic, indexNumThreads, searchNumThreads, bootstrapTimeout, + // channel and connection options + indexNamePrefix, bufferSize, maxRetries, debugMode, + // proxy options + proxyAddress, proxyPassword, proxyUsername, + // certificate options + disableSslVerification, certificateFingerprint, certificatePath, certificateNotRoot + ); + serviceInvoker.AddCommand(service, state, + static async (s, collector, state, ctx) => await s.Index(collector, state.fs, + // endpoint options + state.endpoint, state.environment, state.apiKey, state.username, state.password, + // inference options + state.noSemantic, state.searchNumThreads, state.indexNumThreads, state.bootstrapTimeout, + // channel and connection options + state.indexNamePrefix, state.bufferSize, state.maxRetries, state.debugMode, + // proxy options + state.proxyAddress, state.proxyPassword, state.proxyUsername, + // certificate options + state.disableSslVerification, state.certificateFingerprint, state.certificatePath, state.certificateNotRoot + , ctx) + ); + + return await serviceInvoker.InvokeAsync(ctx); + } +} diff --git a/src/tooling/docs-builder/Program.cs b/src/tooling/docs-builder/Program.cs index 786a54db7..1a8f175b1 100644 --- a/src/tooling/docs-builder/Program.cs +++ b/src/tooling/docs-builder/Program.cs @@ -38,7 +38,7 @@ app.Add("diff"); app.Add("mv"); app.Add("serve"); - +app.Add("index"); //assembler commands @@ -47,6 +47,7 @@ app.Add("assembler bloom-filter"); app.Add("assembler navigation"); app.Add("assembler config"); +app.Add("assembler index"); app.Add("assembler"); app.Add("assemble"); From 44eb49d76c3ebcf60b20431444ef083d990666e5 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 23 Sep 2025 21:11:44 +0200 Subject: [PATCH 2/6] Fix --version and --help being too noisy --- .../AppDefaultsExtensions.cs | 5 ++++- src/Elastic.Documentation/GlobalCommandLine.cs | 15 ++++++++++++--- .../DocumentationTooling.cs | 6 +++++- .../Filters/InfoLoggerFilter.cs | 18 +++++++++++++++--- src/tooling/docs-assembler/Program.cs | 9 ++++++--- .../Filters/CheckForUpdatesFilter.cs | 5 ++++- .../docs-builder/Filters/ReplaceLogFilter.cs | 10 +++++++--- 7 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/Elastic.Documentation.ServiceDefaults/AppDefaultsExtensions.cs b/src/Elastic.Documentation.ServiceDefaults/AppDefaultsExtensions.cs index bcfda4fd1..796bc11bb 100644 --- a/src/Elastic.Documentation.ServiceDefaults/AppDefaultsExtensions.cs +++ b/src/Elastic.Documentation.ServiceDefaults/AppDefaultsExtensions.cs @@ -13,6 +13,8 @@ namespace Elastic.Documentation.ServiceDefaults; +public record CliInvocation(bool IsHelpOrVersion); + public static class AppDefaultsExtensions { public static TBuilder AddDocumentationServiceDefaults(this TBuilder builder) where TBuilder : IHostApplicationBuilder @@ -26,7 +28,7 @@ public static TBuilder AddDocumentationServiceDefaults(this TBuilder b public static TBuilder AddDocumentationServiceDefaults(this TBuilder builder, ref string[] args, LogLevel? defaultLogLevel = null, Action? configure = null) where TBuilder : IHostApplicationBuilder { var logLevel = defaultLogLevel ?? LogLevel.Information; - GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories); + GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories, out var isHelpOrVersion); var services = builder.Services; _ = services @@ -36,6 +38,7 @@ public static TBuilder AddDocumentationServiceDefaults(this TBuilder b configure?.Invoke(s, p); }); _ = builder.Services.AddElasticDocumentationLogging(logLevel); + _ = services.AddSingleton(new CliInvocation(isHelpOrVersion)); return builder.AddServiceDefaults(); } diff --git a/src/Elastic.Documentation/GlobalCommandLine.cs b/src/Elastic.Documentation/GlobalCommandLine.cs index 6de8a22a6..5834e6191 100644 --- a/src/Elastic.Documentation/GlobalCommandLine.cs +++ b/src/Elastic.Documentation/GlobalCommandLine.cs @@ -8,9 +8,15 @@ namespace Elastic.Documentation; public static class GlobalCommandLine { - public static void Process(ref string[] args, ref LogLevel defaultLogLevel, out bool skipPrivateRepositories) + public static void Process( + ref string[] args, + ref LogLevel defaultLogLevel, + out bool skipPrivateRepositories, + out bool isHelpOrVersion + ) { skipPrivateRepositories = false; + isHelpOrVersion = false; var newArgs = new List(); for (var i = 0; i < args.Length; i++) { @@ -22,8 +28,11 @@ public static void Process(ref string[] args, ref LogLevel defaultLogLevel, out } else if (args[i] == "--skip-private-repositories") skipPrivateRepositories = true; - else if (args[i] == "--inject") - skipPrivateRepositories = true; + else if (args[i] is "--help" or "--version") + { + isHelpOrVersion = true; + newArgs.Add(args[i]); + } else newArgs.Add(args[i]); } diff --git a/src/tooling/Elastic.Documentation.Tooling/DocumentationTooling.cs b/src/tooling/Elastic.Documentation.Tooling/DocumentationTooling.cs index 8aa19b226..2d2d52170 100644 --- a/src/tooling/Elastic.Documentation.Tooling/DocumentationTooling.cs +++ b/src/tooling/Elastic.Documentation.Tooling/DocumentationTooling.cs @@ -9,6 +9,7 @@ using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.Versions; using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.ServiceDefaults; using Elastic.Documentation.Tooling.Diagnostics.Console; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -33,9 +34,12 @@ public static TBuilder AddDocumentationToolingDefaults(this TBuilder b { var logFactory = sp.GetRequiredService(); var githubActionsService = sp.GetRequiredService(); + var isHelp = sp.GetRequiredService(); + if (isHelp.IsHelpOrVersion) + return new DiagnosticsCollector([]); return new ConsoleDiagnosticsCollector(logFactory, githubActionsService) { - NoHints = true + NoHints = false }; }) .AddSingleton(sp => diff --git a/src/tooling/Elastic.Documentation.Tooling/Filters/InfoLoggerFilter.cs b/src/tooling/Elastic.Documentation.Tooling/Filters/InfoLoggerFilter.cs index b6bc45ae9..d66284d37 100644 --- a/src/tooling/Elastic.Documentation.Tooling/Filters/InfoLoggerFilter.cs +++ b/src/tooling/Elastic.Documentation.Tooling/Filters/InfoLoggerFilter.cs @@ -5,19 +5,31 @@ using System.Reflection; using ConsoleAppFramework; using Elastic.Documentation.Configuration; +using Elastic.Documentation.ServiceDefaults; using Microsoft.Extensions.Logging; namespace Elastic.Documentation.Tooling.Filters; -public class InfoLoggerFilter(ConsoleAppFilter next, ILogger logger, ConfigurationFileProvider fileProvider) : ConsoleAppFilter(next) +public class InfoLoggerFilter( + ConsoleAppFilter next, + ILogger logger, + ConfigurationFileProvider fileProvider, + CliInvocation cliInvocation +) + : ConsoleAppFilter(next) { public override async Task InvokeAsync(ConsoleAppContext context, Cancel cancellationToken) { + var assemblyVersion = Assembly.GetExecutingAssembly().GetCustomAttributes() + .FirstOrDefault()?.InformationalVersion; + if (cliInvocation.IsHelpOrVersion) + { + await Next.InvokeAsync(context, cancellationToken); + return; + } logger.LogInformation("Configuration source: {ConfigurationSource}", fileProvider.ConfigurationSource.ToStringFast(true)); if (fileProvider.ConfigurationSource == ConfigurationSource.Checkout) logger.LogInformation("Configuration source git reference: {ConfigurationSourceGitReference}", fileProvider.GitReference); - var assemblyVersion = Assembly.GetExecutingAssembly().GetCustomAttributes() - .FirstOrDefault()?.InformationalVersion; logger.LogInformation("Version: {Version}", assemblyVersion); await Next.InvokeAsync(context, cancellationToken); } diff --git a/src/tooling/docs-assembler/Program.cs b/src/tooling/docs-assembler/Program.cs index 71ece40ae..105bd1aca 100644 --- a/src/tooling/docs-assembler/Program.cs +++ b/src/tooling/docs-assembler/Program.cs @@ -46,14 +46,17 @@ await app.RunAsync(args); -internal sealed class ReplaceLogFilter(ConsoleAppFilter next, ILogger logger) +internal sealed class ReplaceLogFilter(ConsoleAppFilter next, ILogger logger, CliInvocation cli) : ConsoleAppFilter(next) { [SuppressMessage("Usage", "CA2254:Template should be a static expression")] public override Task InvokeAsync(ConsoleAppContext context, Cancel cancellationToken) { - ConsoleApp.Log = msg => logger.LogInformation(msg); - ConsoleApp.LogError = msg => logger.LogError(msg); + if (!cli.IsHelpOrVersion) + { + ConsoleApp.Log = msg => logger.LogInformation(msg); + ConsoleApp.LogError = msg => logger.LogError(msg); + } return Next.InvokeAsync(context, cancellationToken); } diff --git a/src/tooling/docs-builder/Filters/CheckForUpdatesFilter.cs b/src/tooling/docs-builder/Filters/CheckForUpdatesFilter.cs index c5de7b8ff..586539b03 100644 --- a/src/tooling/docs-builder/Filters/CheckForUpdatesFilter.cs +++ b/src/tooling/docs-builder/Filters/CheckForUpdatesFilter.cs @@ -6,10 +6,11 @@ using ConsoleAppFramework; using Elastic.Documentation; using Elastic.Documentation.Configuration; +using Elastic.Documentation.ServiceDefaults; namespace Documentation.Builder.Filters; -internal sealed class CheckForUpdatesFilter(ConsoleAppFilter next) : ConsoleAppFilter(next) +internal sealed class CheckForUpdatesFilter(ConsoleAppFilter next, CliInvocation cliInvocation) : ConsoleAppFilter(next) { private readonly FileInfo _stateFile = new(Path.Combine(Paths.ApplicationData.FullName, "docs-build-check.state")); @@ -18,6 +19,8 @@ public override async Task InvokeAsync(ConsoleAppContext context, Cancel ctx) await Next.InvokeAsync(context, ctx); if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("CI"))) return; + if (cliInvocation.IsHelpOrVersion) + return; var latestVersionUrl = await GetLatestVersion(ctx); if (latestVersionUrl is null) diff --git a/src/tooling/docs-builder/Filters/ReplaceLogFilter.cs b/src/tooling/docs-builder/Filters/ReplaceLogFilter.cs index fe93ab36c..afaf41540 100644 --- a/src/tooling/docs-builder/Filters/ReplaceLogFilter.cs +++ b/src/tooling/docs-builder/Filters/ReplaceLogFilter.cs @@ -4,18 +4,22 @@ using System.Diagnostics.CodeAnalysis; using ConsoleAppFramework; +using Elastic.Documentation.ServiceDefaults; using Microsoft.Extensions.Logging; namespace Documentation.Builder.Filters; -internal sealed class ReplaceLogFilter(ConsoleAppFilter next, ILogger logger) +internal sealed class ReplaceLogFilter(ConsoleAppFilter next, ILogger logger, CliInvocation cli) : ConsoleAppFilter(next) { [SuppressMessage("Usage", "CA2254:Template should be a static expression")] public override Task InvokeAsync(ConsoleAppContext context, Cancel cancellationToken) { - ConsoleApp.Log = msg => logger.LogInformation(msg); - ConsoleApp.LogError = msg => logger.LogError(msg); + if (!cli.IsHelpOrVersion) + { + ConsoleApp.Log = msg => logger.LogInformation(msg); + ConsoleApp.LogError = msg => logger.LogError(msg); + } return Next.InvokeAsync(context, cancellationToken); } From aa8267db7d06b7e584108e8b998c8f68fd4f85ce Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 23 Sep 2025 21:36:56 +0200 Subject: [PATCH 3/6] Ensure exporters are start and stopped during isolated builds --- .../Exporters/ElasticsearchMarkdownExporter.cs | 1 + .../Tracking/LocalChangesService.cs | 11 +++++------ .../Building/AssemblerBuildService.cs | 2 -- .../Building/AssemblerBuilder.cs | 1 - .../IsolatedBuildService.cs | 9 ++++++++- src/tooling/docs-builder/Commands/DiffCommands.cs | 2 +- .../{AssemblerIndexCommand.cs => IndexCommand.cs} | 14 ++++++-------- 7 files changed, 21 insertions(+), 19 deletions(-) rename src/tooling/docs-builder/Commands/{AssemblerIndexCommand.cs => IndexCommand.cs} (92%) diff --git a/src/Elastic.Markdown/Exporters/ElasticsearchMarkdownExporter.cs b/src/Elastic.Markdown/Exporters/ElasticsearchMarkdownExporter.cs index a72761529..b01d26b7c 100644 --- a/src/Elastic.Markdown/Exporters/ElasticsearchMarkdownExporter.cs +++ b/src/Elastic.Markdown/Exporters/ElasticsearchMarkdownExporter.cs @@ -27,6 +27,7 @@ public class ElasticsearchMarkdownExporter(ILoggerFactory logFactory, IDiagnosti protected override CatalogIndexChannelOptions NewOptions(DistributedTransport transport) => new(transport) { GetMapping = () => CreateMapping(null), + GetMappingSettings = () => CreateMappingSetting(), IndexFormat = $"{Endpoint.IndexNamePrefix.ToLowerInvariant()}-{indexNamespace.ToLowerInvariant()}-{{0:yyyy.MM.dd.HHmmss}}", ActiveSearchAlias = $"{Endpoint.IndexNamePrefix}-{indexNamespace.ToLowerInvariant()}", }; diff --git a/src/authoring/Elastic.Documentation.Refactor/Tracking/LocalChangesService.cs b/src/authoring/Elastic.Documentation.Refactor/Tracking/LocalChangesService.cs index 0017c2acf..f8fce78db 100644 --- a/src/authoring/Elastic.Documentation.Refactor/Tracking/LocalChangesService.cs +++ b/src/authoring/Elastic.Documentation.Refactor/Tracking/LocalChangesService.cs @@ -19,7 +19,7 @@ IConfigurationContext configurationContext { private readonly ILogger _logger = logFactory.CreateLogger(); - public async Task ValidateRedirects(IDiagnosticsCollector collector, string? path, FileSystem fs, Cancel ctx) + public Task ValidateRedirects(IDiagnosticsCollector collector, string? path, FileSystem fs) { var runningOnCi = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GITHUB_ACTIONS")); @@ -28,21 +28,21 @@ public async Task ValidateRedirects(IDiagnosticsCollector collector, strin if (!redirectFile.Source.Exists) { collector.EmitError(redirectFile.Source, "File does not exist"); - return false; + return Task.FromResult(false); } var redirects = redirectFile.Redirects; if (redirects is null) { collector.EmitError(redirectFile.Source, "It was not possible to parse the redirects file."); - return false; + return Task.FromResult(false); } var root = Paths.DetermineSourceDirectoryRoot(buildContext.DocumentationSourceDirectory); if (root is null) { collector.EmitError(redirectFile.Source, $"Unable to determine the root of the source directory {buildContext.DocumentationSourceDirectory}."); - return false; + return Task.FromResult(false); } var relativePath = Path.GetRelativePath(root.FullName, buildContext.DocumentationSourceDirectory.FullName); _logger.LogInformation("Using relative path {RelativePath} for validating changes", relativePath); @@ -87,7 +87,6 @@ public async Task ValidateRedirects(IDiagnosticsCollector collector, strin _logger.LogInformation("Found {Count} changes that still require updates to: {RedirectFile}", missingCount, relativeRedirectFile); } - await collector.StopAsync(ctx); - return collector.Errors == 0; + return Task.FromResult(collector.Errors == 0); } } diff --git a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs index 271ff7567..3ca9e0061 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs @@ -93,8 +93,6 @@ public async Task BuildAll(IDiagnosticsCollector collector, bool? strict, sitemapBuilder.Generate(); } - await collector.StopAsync(ctx); - _logger.LogInformation("Finished building and exporting exporters {Exporters}", exporters); return strict.Value ? collector.Errors + collector.Warnings == 0 : collector.Errors == 0; diff --git a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs index 84c6a3747..df15b6bf6 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs @@ -42,7 +42,6 @@ public async Task BuildAllAsync(PublishEnvironment environment, FrozenDictionary var redirects = new Dictionary(); var markdownExporters = exportOptions.CreateMarkdownExporters(logFactory, context, environment.Name); - var tasks = markdownExporters.Select(async e => await e.StartAsync(ctx)); await Task.WhenAll(tasks); diff --git a/src/services/Elastic.Documentation.Isolated/IsolatedBuildService.cs b/src/services/Elastic.Documentation.Isolated/IsolatedBuildService.cs index 70d6efa11..335d7441c 100644 --- a/src/services/Elastic.Documentation.Isolated/IsolatedBuildService.cs +++ b/src/services/Elastic.Documentation.Isolated/IsolatedBuildService.cs @@ -114,6 +114,10 @@ public async Task Build( var markdownExporters = exporters.CreateMarkdownExporters(logFactory, context, "isolated"); + var tasks = markdownExporters.Select(async e => await e.StartAsync(ctx)); + await Task.WhenAll(tasks); + + var generator = new DocumentationGenerator(set, logFactory, null, null, markdownExporters.ToArray()); _ = await generator.GenerateAll(ctx); @@ -123,7 +127,10 @@ public async Task Build( if (runningOnCi) await githubActionsService.SetOutputAsync("landing-page-path", set.FirstInterestingUrl); - await collector.StopAsync(ctx); + tasks = markdownExporters.Select(async e => await e.StopAsync(ctx)); + await Task.WhenAll(tasks); + _logger.LogInformation("Finished building and exporting exporters {Exporters}", exporters); + return strict.Value ? context.Collector.Errors + context.Collector.Warnings == 0 : context.Collector.Errors == 0; } } diff --git a/src/tooling/docs-builder/Commands/DiffCommands.cs b/src/tooling/docs-builder/Commands/DiffCommands.cs index eda899d9c..895040cf8 100644 --- a/src/tooling/docs-builder/Commands/DiffCommands.cs +++ b/src/tooling/docs-builder/Commands/DiffCommands.cs @@ -32,7 +32,7 @@ public async Task ValidateRedirects(string? path = null, Cancel ctx = defau var fs = new FileSystem(); serviceInvoker.AddCommand(service, (path, fs), - async static (s, collector, state, ctx) => await s.ValidateRedirects(collector, state.path, state.fs, ctx) + async static (s, collector, state, _) => await s.ValidateRedirects(collector, state.path, state.fs) ); return await serviceInvoker.InvokeAsync(ctx); } diff --git a/src/tooling/docs-builder/Commands/AssemblerIndexCommand.cs b/src/tooling/docs-builder/Commands/IndexCommand.cs similarity index 92% rename from src/tooling/docs-builder/Commands/AssemblerIndexCommand.cs rename to src/tooling/docs-builder/Commands/IndexCommand.cs index e93332cd5..739351317 100644 --- a/src/tooling/docs-builder/Commands/AssemblerIndexCommand.cs +++ b/src/tooling/docs-builder/Commands/IndexCommand.cs @@ -5,9 +5,7 @@ using System.IO.Abstractions; using Actions.Core.Services; using ConsoleAppFramework; -using Elastic.Documentation.Assembler.Indexing; using Elastic.Documentation.Configuration; -using Elastic.Documentation.Configuration.Assembler; using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Isolated; using Elastic.Documentation.Services; @@ -26,7 +24,7 @@ ICoreService githubActionsService /// Index a single documentation set to Elasticsearch, calls `docs-builder --exporters elasticsearch`. Exposes more options /// /// Elasticsearch endpoint, alternatively set env DOCUMENTATION_ELASTIC_URL - /// The --environment used to clone ends up being part of the index name + /// path to the documentation folder, defaults to pwd. /// Elasticsearch API key, alternatively set env DOCUMENTATION_ELASTIC_APIKEY /// Elasticsearch username (basic auth), alternatively set env DOCUMENTATION_ELASTIC_USERNAME /// Elasticsearch password (basic auth), alternatively set env DOCUMENTATION_ELASTIC_PASSWORD @@ -50,7 +48,7 @@ ICoreService githubActionsService [Command("")] public async Task Index( [Argument] string? endpoint = null, - string? environment = null, + string? path = null, string? apiKey = null, string? username = null, string? password = null, @@ -87,9 +85,9 @@ public async Task Index( await using var serviceInvoker = new ServiceInvoker(collector); var fs = new FileSystem(); var service = new IsolatedIndexService(logFactory, configurationContext, githubActionsService); - var state = (fs, + var state = (fs, path, // endpoint options - endpoint, environment, apiKey, username, password, + endpoint, apiKey, username, password, // inference options noSemantic, indexNumThreads, searchNumThreads, bootstrapTimeout, // channel and connection options @@ -100,9 +98,9 @@ public async Task Index( disableSslVerification, certificateFingerprint, certificatePath, certificateNotRoot ); serviceInvoker.AddCommand(service, state, - static async (s, collector, state, ctx) => await s.Index(collector, state.fs, + static async (s, collector, state, ctx) => await s.Index(collector, state.fs, state.path, // endpoint options - state.endpoint, state.environment, state.apiKey, state.username, state.password, + state.endpoint, state.apiKey, state.username, state.password, // inference options state.noSemantic, state.searchNumThreads, state.indexNumThreads, state.bootstrapTimeout, // channel and connection options From f7c6f514ce5d1d6483131d673845408ae4801ed0 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 23 Sep 2025 21:51:26 +0200 Subject: [PATCH 4/6] Improve export logging a tad --- .../Exporters/ElasticsearchMarkdownExporter.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Elastic.Markdown/Exporters/ElasticsearchMarkdownExporter.cs b/src/Elastic.Markdown/Exporters/ElasticsearchMarkdownExporter.cs index b01d26b7c..b3747212d 100644 --- a/src/Elastic.Markdown/Exporters/ElasticsearchMarkdownExporter.cs +++ b/src/Elastic.Markdown/Exporters/ElasticsearchMarkdownExporter.cs @@ -187,6 +187,7 @@ public async ValueTask StartAsync(Cancel ctx = default) //The max num threads per allocated node, from testing its best to limit our max concurrency //producing to this number as well var options = NewOptions(transport); + var i = 0; options.BufferOptions = new BufferOptions { OutboundBufferMaxSize = Endpoint.BufferSize, @@ -194,7 +195,12 @@ public async ValueTask StartAsync(Cancel ctx = default) ExportMaxRetries = Endpoint.MaxRetries, }; options.SerializerContext = SourceGenerationContext.Default; - options.ExportBufferCallback = () => _logger.LogInformation("Exported buffer to Elasticsearch"); + options.ExportBufferCallback = () => + { + var count = Interlocked.Increment(ref i); + _logger.LogInformation("Exported {Count} documents to Elasticsearch index {Format}", + count * Endpoint.BufferSize, options.IndexFormat); + }; options.ExportExceptionCallback = e => _logger.LogError(e, "Failed to export document"); options.ServerRejectionCallback = items => _logger.LogInformation("Server rejection: {Rejection}", items.First().Item2); _channel = NewChannel(options); From 2344259c6583c573549fae6a6a98463dd63b9d96 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 23 Sep 2025 22:08:10 +0200 Subject: [PATCH 5/6] Ensure assemble build shows no hints but we now expose --show-hints to enable it --- .../Diagnostics/DiagnosticsCollector.cs | 2 +- .../Diagnostics/IDiagnosticsCollector.cs | 2 ++ .../Building/AssemblerBuildService.cs | 12 ++++++++++-- .../Indexing/AssemblerIndexService.cs | 2 +- .../DocumentationTooling.cs | 5 +---- .../docs-assembler/Cli/RepositoryCommands.cs | 3 ++- .../Commands/Assembler/AssemblerCommands.cs | 16 +++++++++++----- 7 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/Elastic.Documentation/Diagnostics/DiagnosticsCollector.cs b/src/Elastic.Documentation/Diagnostics/DiagnosticsCollector.cs index 9917ae8da..5a0914d9d 100644 --- a/src/Elastic.Documentation/Diagnostics/DiagnosticsCollector.cs +++ b/src/Elastic.Documentation/Diagnostics/DiagnosticsCollector.cs @@ -29,7 +29,7 @@ public class DiagnosticsCollector(IReadOnlyCollection output public ConcurrentBag CrossLinks { get; } = []; - public bool NoHints { get; init; } + public bool NoHints { get; set; } public DiagnosticsCollector StartAsync(Cancel ctx) { diff --git a/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs b/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs index 2416509f4..6e2c2c361 100644 --- a/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs +++ b/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs @@ -14,6 +14,8 @@ public interface IDiagnosticsCollector : IAsyncDisposable, IHostedService int Errors { get; } int Hints { get; } + bool NoHints { get; set; } + DiagnosticsChannel Channel { get; } ConcurrentBag CrossLinks { get; } HashSet OffendingFiles { get; } diff --git a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs index 3ca9e0061..81d548de3 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs @@ -11,7 +11,6 @@ using Elastic.Documentation.Configuration.Assembler; using Elastic.Documentation.Configuration.Navigation; using Elastic.Documentation.Diagnostics; -using Elastic.Documentation.Legacy; using Elastic.Documentation.LegacyDocs; using Elastic.Documentation.Services; using Microsoft.Extensions.Logging; @@ -27,8 +26,17 @@ ICoreService githubActionsService { private readonly ILogger _logger = logFactory.CreateLogger(); - public async Task BuildAll(IDiagnosticsCollector collector, bool? strict, string? environment, bool? metadataOnly, IReadOnlySet? exporters, FileSystem fs, Cancel ctx) + public async Task BuildAll( + IDiagnosticsCollector collector, + bool? strict, string? environment, + bool? metadataOnly, + bool? showHints, + IReadOnlySet? exporters, + FileSystem fs, + Cancel ctx + ) { + collector.NoHints = !showHints.GetValueOrDefault(false); strict ??= false; exporters ??= metadataOnly.GetValueOrDefault(false) ? ExportOptions.MetadataOnly : ExportOptions.Default; // ensure we never generate a documentation state for assembler builds diff --git a/src/services/Elastic.Documentation.Assembler/Indexing/AssemblerIndexService.cs b/src/services/Elastic.Documentation.Assembler/Indexing/AssemblerIndexService.cs index 0a8b6bf96..a892816e1 100644 --- a/src/services/Elastic.Documentation.Assembler/Indexing/AssemblerIndexService.cs +++ b/src/services/Elastic.Documentation.Assembler/Indexing/AssemblerIndexService.cs @@ -133,6 +133,6 @@ public async Task Index(IDiagnosticsCollector collector, var exporters = new HashSet { noSemantic.GetValueOrDefault(false) ? ElasticsearchNoSemantic : Elasticsearch }; - return await BuildAll(collector, strict: false, environment, metadataOnly: true, exporters, fileSystem, ctx); + return await BuildAll(collector, strict: false, environment, metadataOnly: true, showHints: false, exporters, fileSystem, ctx); } } diff --git a/src/tooling/Elastic.Documentation.Tooling/DocumentationTooling.cs b/src/tooling/Elastic.Documentation.Tooling/DocumentationTooling.cs index 2d2d52170..698268912 100644 --- a/src/tooling/Elastic.Documentation.Tooling/DocumentationTooling.cs +++ b/src/tooling/Elastic.Documentation.Tooling/DocumentationTooling.cs @@ -37,10 +37,7 @@ public static TBuilder AddDocumentationToolingDefaults(this TBuilder b var isHelp = sp.GetRequiredService(); if (isHelp.IsHelpOrVersion) return new DiagnosticsCollector([]); - return new ConsoleDiagnosticsCollector(logFactory, githubActionsService) - { - NoHints = false - }; + return new ConsoleDiagnosticsCollector(logFactory, githubActionsService); }) .AddSingleton(sp => { diff --git a/src/tooling/docs-assembler/Cli/RepositoryCommands.cs b/src/tooling/docs-assembler/Cli/RepositoryCommands.cs index 683823ac9..56016bbfa 100644 --- a/src/tooling/docs-assembler/Cli/RepositoryCommands.cs +++ b/src/tooling/docs-assembler/Cli/RepositoryCommands.cs @@ -90,7 +90,8 @@ public async Task BuildAll( var fs = new FileSystem(); var service = new AssemblerBuildService(logFactory, assemblyConfiguration, configurationContext, githubActionsService); serviceInvoker.AddCommand(service, (strict, environment, metadataOnly, exporters, fs), strict ?? false, - static async (s, collector, state, ctx) => await s.BuildAll(collector, state.strict, state.environment, state.metadataOnly, state.exporters, state.fs, ctx) + static async (s, collector, state, ctx) => + await s.BuildAll(collector, state.strict, state.environment, state.metadataOnly, false, state.exporters, state.fs, ctx) ); return await serviceInvoker.InvokeAsync(ctx); diff --git a/src/tooling/docs-builder/Commands/Assembler/AssemblerCommands.cs b/src/tooling/docs-builder/Commands/Assembler/AssemblerCommands.cs index ca5a84d88..66940462b 100644 --- a/src/tooling/docs-builder/Commands/Assembler/AssemblerCommands.cs +++ b/src/tooling/docs-builder/Commands/Assembler/AssemblerCommands.cs @@ -32,6 +32,7 @@ ICoreService githubActionsService /// If true, fetch the latest commit of the branch instead of the link registry entry ref /// If true, assume the repository folder already exists on disk assume it's cloned already, primarily used for testing /// Only emit documentation metadata to output, ignored if 'exporters' is also set + /// Show hints from all documentation sets during assembler build /// Set available exporters: /// html, es, config, links, state, llm, redirect, metadata, none. /// Defaults to (html, config, links, state, redirect) or 'default'. @@ -39,12 +40,13 @@ ICoreService githubActionsService /// Serve the documentation on port 4000 after succesful build /// [Command("")] - public async Task CloneAll( + public async Task CloneAndBuild( bool? strict = null, string? environment = null, bool? fetchLatest = null, bool? assumeCloned = null, bool? metadataOnly = null, + bool? showHints = null, [ExporterParser] IReadOnlySet? exporters = null, bool serve = false, Cancel ctx = default @@ -59,8 +61,9 @@ static async (s, collector, state, ctx) => await s.CloneAll(collector, state.str var buildService = new AssemblerBuildService(logFactory, assemblyConfiguration, configurationContext, githubActionsService); var fs = new FileSystem(); - serviceInvoker.AddCommand(buildService, (strict, environment, metadataOnly, exporters, fs), strict ?? false, - static async (s, collector, state, ctx) => await s.BuildAll(collector, state.strict, state.environment, state.metadataOnly, state.exporters, state.fs, ctx) + serviceInvoker.AddCommand(buildService, (strict, environment, metadataOnly, showHints, exporters, fs), strict ?? false, + static async (s, collector, state, ctx) => + await s.BuildAll(collector, state.strict, state.environment, state.metadataOnly, state.showHints, state.exporters, state.fs, ctx) ); var result = await serviceInvoker.InvokeAsync(ctx); @@ -114,6 +117,7 @@ static async (s, collector, state, ctx) => await s.CloneAll(collector, state.str /// Treat warnings as errors and fail the build on warnings /// The environment to build /// Only emit documentation metadata to output, ignored if 'exporters' is also set + /// Show hints from all documentation sets during assembler build /// Set available exporters: /// html, es, config, links, state, llm, redirect, metadata, none. /// Defaults to (html, config, links, state, redirect) or 'default'. @@ -124,6 +128,7 @@ public async Task BuildAll( bool? strict = null, string? environment = null, bool? metadataOnly = null, + bool? showHints = null, [ExporterParser] IReadOnlySet? exporters = null, Cancel ctx = default ) @@ -132,8 +137,9 @@ public async Task BuildAll( var fs = new FileSystem(); var service = new AssemblerBuildService(logFactory, assemblyConfiguration, configurationContext, githubActionsService); - serviceInvoker.AddCommand(service, (strict, environment, metadataOnly, exporters, fs), strict ?? false, - static async (s, collector, state, ctx) => await s.BuildAll(collector, state.strict, state.environment, state.metadataOnly, state.exporters, state.fs, ctx) + serviceInvoker.AddCommand(service, (strict, environment, metadataOnly, showHints, exporters, fs), strict ?? false, + static async (s, collector, state, ctx) => + await s.BuildAll(collector, state.strict, state.environment, state.metadataOnly, state.showHints, state.exporters, state.fs, ctx) ); return await serviceInvoker.InvokeAsync(ctx); From 51a3e223ef72cf4640f1e4d4d0b5bf5142a99be6 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 23 Sep 2025 22:24:50 +0200 Subject: [PATCH 6/6] build failures --- aspire/AppHost.cs | 2 +- tests/Elastic.Markdown.Tests/FileSystemExtensionsTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aspire/AppHost.cs b/aspire/AppHost.cs index a3846cf2e..6e4eed980 100644 --- a/aspire/AppHost.cs +++ b/aspire/AppHost.cs @@ -13,7 +13,7 @@ // ReSharper disable NotAccessedVariable var logLevel = LogLevel.Information; -GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories); +GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories, out _); var globalArguments = new List(); if (skipPrivateRepositories) globalArguments.Add("--skip-private-repositories"); diff --git a/tests/Elastic.Markdown.Tests/FileSystemExtensionsTests.cs b/tests/Elastic.Markdown.Tests/FileSystemExtensionsTests.cs index a8db8e8b6..fd624b1c7 100644 --- a/tests/Elastic.Markdown.Tests/FileSystemExtensionsTests.cs +++ b/tests/Elastic.Markdown.Tests/FileSystemExtensionsTests.cs @@ -13,7 +13,7 @@ namespace Elastic.Markdown.Tests; -public class FileSystemExtensionsTest(ITestOutputHelper output) +public class FileSystemExtensionsTest { [Fact] public void IsSubPathOfTests()