diff --git a/src/docs-assembler/Cli/RepositoryCommands.cs b/src/docs-assembler/Cli/RepositoryCommands.cs index 3c283d59f..022b96a17 100644 --- a/src/docs-assembler/Cli/RepositoryCommands.cs +++ b/src/docs-assembler/Cli/RepositoryCommands.cs @@ -2,15 +2,25 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.Collections.Concurrent; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO.Abstractions; +using System.Net.Mime; using Actions.Core.Services; +using Amazon.S3; +using Amazon.S3.Model; using ConsoleAppFramework; using Documentation.Assembler.Building; +using Documentation.Assembler.Configuration; using Documentation.Assembler.Mapping; using Documentation.Assembler.Navigation; using Documentation.Assembler.Sourcing; using Elastic.Documentation.Tooling.Diagnostics.Console; +using Elastic.Markdown; +using Elastic.Markdown.Exporters; +using Elastic.Markdown.IO; +using Elastic.Markdown.IO.State; using Microsoft.Extensions.Logging; namespace Documentation.Assembler.Cli; @@ -117,4 +127,68 @@ public async Task BuildAll( return collector.Errors + collector.Warnings; return collector.Errors; } + + /// The content source. "current" or "next" + /// + [Command("update-all-link-reference")] + public async Task UpdateLinkIndexAll(ContentSource contentSource, Cancel ctx = default) + { + var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService); + // The environment ist not relevant here. + // It's only used to get the list of repositories. + var assembleContext = new AssembleContext("prod", collector, new FileSystem(), new FileSystem(), null, null); + var cloner = new RepositorySourcer(logger, assembleContext.CheckoutDirectory, new FileSystem(), collector); + var dict = new ConcurrentDictionary(); + var repositories = new Dictionary(assembleContext.Configuration.ReferenceRepositories) + { + { NarrativeRepository.RepositoryName, assembleContext.Configuration.Narrative } + }; + await Parallel.ForEachAsync(repositories, + new ParallelOptions + { + CancellationToken = ctx, + MaxDegreeOfParallelism = Environment.ProcessorCount + }, async (kv, c) => + { + try + { + var name = kv.Key.Trim(); + var checkout = cloner.CloneOrUpdateRepository(kv.Value, name, kv.Value.GetBranch(contentSource), dict); + + var docsMetadataPath = Path.Combine(checkout.Directory.FullName, ".docs-metadata"); + + var context = new BuildContext( + collector, + new FileSystem(), + new FileSystem(), + checkout.Directory.FullName, + docsMetadataPath + ); + var set = new DocumentationSet(context, logger); + var generator = new DocumentationGenerator(set, logger, null, null, new NoopDocumentationFileExporter()); + await generator.GenerateAll(c); + + IAmazonS3 s3Client = new AmazonS3Client(); + const string bucketName = "elastic-docs-link-index"; + var linksJsonPath = Path.Combine(docsMetadataPath, "links.json"); + var content = await File.ReadAllTextAsync(linksJsonPath, c); + var putObjectRequest = new PutObjectRequest + { + BucketName = bucketName, + Key = $"elastic/{checkout.Repository.Name}/{checkout.Repository.GetBranch(contentSource)}/links.json", + ContentBody = content, + ContentType = MediaTypeNames.Application.Json, + ChecksumAlgorithm = ChecksumAlgorithm.SHA256 + }; + var response = await s3Client.PutObjectAsync(putObjectRequest, c); + if (response.HttpStatusCode != System.Net.HttpStatusCode.OK) + collector.EmitError(linksJsonPath, $"Failed to upload {putObjectRequest.Key} to S3"); + } + catch (Exception e) + { + collector.EmitError(kv.Key, $"Failed to update link index for {kv.Key}: {e.Message}", e); + } + }).ConfigureAwait(false); + return collector.Errors > 0 ? 1 : 0; + } } diff --git a/src/docs-assembler/Configuration/Repository.cs b/src/docs-assembler/Configuration/Repository.cs index 1d783e0af..e928ca355 100644 --- a/src/docs-assembler/Configuration/Repository.cs +++ b/src/docs-assembler/Configuration/Repository.cs @@ -2,6 +2,7 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using Elastic.Markdown.IO.State; using YamlDotNet.Serialization; namespace Documentation.Assembler.Configuration; @@ -31,4 +32,11 @@ public record Repository [YamlMember(Alias = "skip")] public bool Skip { get; set; } + + public string GetBranch(ContentSource contentSource) => contentSource switch + { + ContentSource.Current => GitReferenceCurrent, + ContentSource.Next => GitReferenceNext, + _ => throw new ArgumentException($"The content source {contentSource} is not supported.", nameof(contentSource)) + }; } diff --git a/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs b/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs index 1220ef3d4..d3ed69ebe 100644 --- a/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs +++ b/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs @@ -7,6 +7,8 @@ using System.Diagnostics.CodeAnalysis; using System.IO.Abstractions; using Documentation.Assembler.Configuration; +using Elastic.Documentation.Tooling.Diagnostics.Console; +using Elastic.Markdown.Diagnostics; using Elastic.Markdown.IO; using Elastic.Markdown.IO.State; using Microsoft.Extensions.Logging; @@ -22,6 +24,8 @@ public class AssemblerRepositorySourcer(ILoggerFactory logger, AssembleContext c private AssemblyConfiguration Configuration => context.Configuration; private PublishEnvironment PublishEnvironment => context.Environment; + private RepositorySourcer RepositorySourcer => new(logger, context.CheckoutDirectory, context.ReadFileSystem, context.Collector); + public IReadOnlyCollection GetAll() { var fs = context.ReadFileSystem; @@ -52,15 +56,23 @@ public async Task> AcquireAllLatest(Cancel ctx = d PublishEnvironment.ContentSource.ToStringFast(true) ); - var dict = new ConcurrentDictionary(); - var checkouts = new ConcurrentBag(); + var repositories = new Dictionary(Configuration.ReferenceRepositories) + { + { NarrativeRepository.RepositoryName, Configuration.Narrative } + }; + return await RepositorySourcer.AcquireAllLatest(repositories, PublishEnvironment.ContentSource, ctx); + } +} - _logger.LogInformation("Cloning narrative content: {Repository}", NarrativeRepository.RepositoryName); - var checkout = CloneOrUpdateRepository(Configuration.Narrative, NarrativeRepository.RepositoryName, dict); - checkouts.Add(checkout); +public class RepositorySourcer(ILoggerFactory logger, IDirectoryInfo checkoutDirectory, IFileSystem readFileSystem, DiagnosticsCollector collector) +{ + private readonly ILogger _logger = logger.CreateLogger(); - _logger.LogInformation("Cloning {ReferenceRepositoryCount} repositories", Configuration.ReferenceRepositories.Count); - await Parallel.ForEachAsync(Configuration.ReferenceRepositories, + public async Task> AcquireAllLatest(Dictionary repositories, ContentSource source, Cancel ctx = default) + { + var dict = new ConcurrentDictionary(); + var checkouts = new ConcurrentBag(); + await Parallel.ForEachAsync(repositories, new ParallelOptions { CancellationToken = ctx, @@ -70,26 +82,21 @@ await Parallel.ForEachAsync(Configuration.ReferenceRepositories, await Task.Run(() => { var name = kv.Key.Trim(); - var clone = CloneOrUpdateRepository(kv.Value, name, dict); + var repo = kv.Value; + var clone = CloneOrUpdateRepository(kv.Value, name, repo.GetBranch(source), dict); checkouts.Add(clone); }, c); }).ConfigureAwait(false); - foreach (var kv in dict.OrderBy(kv => kv.Value.Elapsed)) - _logger.LogInformation("-> took: {Elapsed}\t{RepositoryBranch}", kv.Key, kv.Value.Elapsed); - return checkouts.ToList().AsReadOnly(); } - private Checkout CloneOrUpdateRepository(Repository repository, string name, ConcurrentDictionary dict) + public Checkout CloneOrUpdateRepository(Repository repository, string name, string branch, ConcurrentDictionary dict) { - var fs = context.ReadFileSystem; - var checkoutFolder = fs.DirectoryInfo.New(Path.Combine(context.CheckoutDirectory.FullName, name)); + var fs = readFileSystem; + var checkoutFolder = fs.DirectoryInfo.New(Path.Combine(checkoutDirectory.FullName, name)); var relativePath = Path.GetRelativePath(Paths.WorkingDirectoryRoot.FullName, checkoutFolder.FullName); var sw = Stopwatch.StartNew(); - var branch = PublishEnvironment.ContentSource == ContentSource.Next - ? repository.GitReferenceNext - : repository.GitReferenceCurrent; _ = dict.AddOrUpdate($"{name} ({branch})", sw, (_, _) => sw); @@ -140,22 +147,23 @@ private string CheckoutFromScratch(Repository repository, string name, string br IDirectoryInfo checkoutFolder) { _logger.LogInformation("Checkout: {Name}\t{Branch}\t{RelativePath}", name, branch, relativePath); - if (repository.CheckoutStrategy == "full") + switch (repository.CheckoutStrategy) { - Exec("git", "clone", repository.Origin, checkoutFolder.FullName, - "--depth", "1", "--single-branch", - "--branch", branch - ); - } - else if (repository.CheckoutStrategy == "partial") - { - Exec( - "git", "clone", "--filter=blob:none", "--no-checkout", repository.Origin, checkoutFolder.FullName - ); - - ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "--cone"); - ExecIn(checkoutFolder, "git", "checkout", branch); - ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "docs"); + case "full": + Exec("git", "clone", repository.Origin, checkoutFolder.FullName, + "--depth", "1", "--single-branch", + "--branch", branch + ); + break; + case "partial": + Exec( + "git", "clone", "--filter=blob:none", "--no-checkout", repository.Origin, checkoutFolder.FullName + ); + + ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "--cone"); + ExecIn(checkoutFolder, "git", "checkout", branch); + ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "docs"); + break; } return Capture(checkoutFolder, "git", "rev-parse", "HEAD"); @@ -171,7 +179,7 @@ private void ExecIn(IDirectoryInfo? workingDirectory, string binary, params stri }; var result = Proc.Exec(arguments); if (result != 0) - context.Collector.EmitError("", $"Exit code: {result} while executing {binary} {string.Join(" ", args)} in {workingDirectory}"); + collector.EmitError("", $"Exit code: {result} while executing {binary} {string.Join(" ", args)} in {workingDirectory}"); } // ReSharper disable once UnusedMember.Local @@ -203,11 +211,12 @@ string CaptureOutput() }; var result = Proc.Start(arguments); if (result.ExitCode != 0) - context.Collector.EmitError("", $"Exit code: {result.ExitCode} while executing {binary} {string.Join(" ", args)} in {workingDirectory}"); + collector.EmitError("", $"Exit code: {result.ExitCode} while executing {binary} {string.Join(" ", args)} in {workingDirectory}"); var line = result.ConsoleOut.FirstOrDefault()?.Line ?? throw new Exception($"No output captured for {binary}: {workingDirectory}"); return line; } } + } public class NoopConsoleWriter : IConsoleOutWriter diff --git a/src/docs-builder/Cli/Commands.cs b/src/docs-builder/Cli/Commands.cs index a2415f5f2..df165f872 100644 --- a/src/docs-builder/Cli/Commands.cs +++ b/src/docs-builder/Cli/Commands.cs @@ -102,7 +102,7 @@ public async Task Generate( var runningOnCi = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GITHUB_ACTIONS")); BuildContext context; - Uri? canonicalBaseUri = null; + Uri? canonicalBaseUri; if (canonicalBaseUrl is null) canonicalBaseUri = new Uri("https://docs-v3-preview.elastic.dev");