diff --git a/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs b/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs index 9808ce11e..14bba092c 100644 --- a/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs +++ b/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs @@ -5,12 +5,41 @@ using System.Collections.Frozen; using System.Diagnostics.CodeAnalysis; using System.Text.Json; +using System.Text.Json.Serialization; using Elastic.Markdown.IO.Configuration; using Elastic.Markdown.IO.State; using Microsoft.Extensions.Logging; namespace Elastic.Markdown.CrossLinks; + +public record LinkIndex +{ + [JsonPropertyName("repositories")] + public required Dictionary> Repositories { get; init; } + + public static LinkIndex Deserialize(string json) => + JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkIndex)!; + + public static string Serialize(LinkIndex index) => + JsonSerializer.Serialize(index, SourceGenerationContext.Default.LinkIndex); +} + +public record LinkIndexEntry +{ + [JsonPropertyName("repository")] + public required string Repository { get; init; } + + [JsonPropertyName("path")] + public required string Path { get; init; } + + [JsonPropertyName("branch")] + public required string Branch { get; init; } + + [JsonPropertyName("etag")] + public required string ETag { get; init; } +} + public interface ICrossLinkResolver { Task FetchLinks(); diff --git a/src/Elastic.Markdown/SourceGenerationContext.cs b/src/Elastic.Markdown/SourceGenerationContext.cs index 5ae58eb99..7cb72296d 100644 --- a/src/Elastic.Markdown/SourceGenerationContext.cs +++ b/src/Elastic.Markdown/SourceGenerationContext.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using System.Text.Json.Serialization; +using Elastic.Markdown.CrossLinks; using Elastic.Markdown.IO; using Elastic.Markdown.IO.Discovery; using Elastic.Markdown.IO.State; @@ -15,4 +16,6 @@ namespace Elastic.Markdown; [JsonSerializable(typeof(GenerationState))] [JsonSerializable(typeof(LinkReference))] [JsonSerializable(typeof(GitCheckoutInformation))] +[JsonSerializable(typeof(LinkIndex))] +[JsonSerializable(typeof(LinkIndexEntry))] internal partial class SourceGenerationContext : JsonSerializerContext; diff --git a/src/docs-assembler/Cli/LinkCommands.cs b/src/docs-assembler/Cli/LinkCommands.cs new file mode 100644 index 000000000..0ab50b88a --- /dev/null +++ b/src/docs-assembler/Cli/LinkCommands.cs @@ -0,0 +1,87 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text; +using Actions.Core.Services; +using Amazon.S3; +using Amazon.S3.Model; +using ConsoleAppFramework; +using Elastic.Markdown.CrossLinks; +using Microsoft.Extensions.Logging; + +namespace Documentation.Assembler.Cli; + +internal class LinkCommands(ILoggerFactory logger) +{ + private void AssignOutputLogger() + { + var log = logger.CreateLogger(); + ConsoleApp.Log = msg => log.LogInformation(msg); + ConsoleApp.LogError = msg => log.LogError(msg); + } + + /// + /// Create an index.json file from all discovered links.json files in our S3 bucket + /// + /// + [Command("links create-index")] + public async Task CreateLinkIndex(Cancel ctx = default) + { + AssignOutputLogger(); + + IAmazonS3 client = new AmazonS3Client(); + var bucketName = "elastic-docs-link-index"; + var request = new ListObjectsV2Request { BucketName = bucketName, MaxKeys = 5 }; + + Console.WriteLine("--------------------------------------"); + Console.WriteLine($"Listing the contents of {bucketName}:"); + Console.WriteLine("--------------------------------------"); + + + var linkIndex = new LinkIndex { Repositories = new Dictionary>() }; + try + { + ListObjectsV2Response response; + do + { + response = await client.ListObjectsV2Async(request, ctx); + foreach (var obj in response.S3Objects) + { + if (!obj.Key.StartsWith("elastic/")) + continue; + + var tokens = obj.Key.Split('/'); + if (tokens.Length < 3) + continue; + + var repository = tokens[1]; + var branch = tokens[2]; + + var entry = new LinkIndexEntry { Repository = repository, Branch = branch, ETag = obj.ETag.Trim('"'), Path = obj.Key }; + if (linkIndex.Repositories.TryGetValue(repository, out var existingEntry)) + existingEntry[branch] = entry; + else + linkIndex.Repositories.Add(repository, new Dictionary { { branch, entry } }); + Console.WriteLine(entry); + } + + // If the response is truncated, set the request ContinuationToken + // from the NextContinuationToken property of the response. + request.ContinuationToken = response.NextContinuationToken; + } while (response.IsTruncated); + } + catch (AmazonS3Exception ex) + { + Console.WriteLine($"Error encountered on server. Message:'{ex.Message}' getting list of objects."); + } + + var json = LinkIndex.Serialize(linkIndex); + Console.WriteLine(json); + + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + await client.UploadObjectFromStreamAsync(bucketName, "link-index.json", stream, new Dictionary(), ctx); + + Console.WriteLine("Uploaded latest link-index.json"); + } +} diff --git a/src/docs-assembler/Program.cs b/src/docs-assembler/Program.cs index cf4ff2631..839f9282b 100644 --- a/src/docs-assembler/Program.cs +++ b/src/docs-assembler/Program.cs @@ -4,20 +4,47 @@ using System.Collections.Concurrent; using System.Diagnostics; +using Actions.Core.Extensions; using ConsoleAppFramework; using Documentation.Assembler; using Documentation.Assembler.Cli; +using Elastic.Markdown.Diagnostics; using Elastic.Markdown.IO; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using ProcNet; using ProcNet.Std; var configFile = Path.Combine(Paths.Root.FullName, "src/docs-assembler/conf.yml"); var config = AssemblyConfiguration.Deserialize(File.ReadAllText(configFile)); +var services = new ServiceCollection(); +services.AddGitHubActionsCore(); +services.AddLogging(x => +{ + x.ClearProviders(); + x.SetMinimumLevel(LogLevel.Information); + x.AddSimpleConsole(c => + { + c.SingleLine = true; + c.IncludeScopes = true; + c.UseUtcTimestamp = true; + c.TimestampFormat = Environment.UserInteractive ? ":: " : "[yyyy-MM-ddTHH:mm:ss] "; + }); +}); +services.AddSingleton(); +services.AddSingleton(); + +await using var serviceProvider = services.BuildServiceProvider(); +ConsoleApp.ServiceProvider = serviceProvider; + + var app = ConsoleApp.Create(); app.UseFilter(); app.UseFilter(); +app.Add(); + // would love to use libgit2 so there is no git dependency but // libgit2 is magnitudes slower to clone repositories https://github.com/libgit2/libgit2/issues/4674 app.Add("clone-all", async Task (CancellationToken ctx) => diff --git a/src/docs-assembler/docs-assembler.csproj b/src/docs-assembler/docs-assembler.csproj index 0e1d83e28..5caeaa4a2 100644 --- a/src/docs-assembler/docs-assembler.csproj +++ b/src/docs-assembler/docs-assembler.csproj @@ -16,6 +16,7 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive