diff --git a/src/Elastic.Markdown/Assets/styles.css b/src/Elastic.Markdown/Assets/styles.css index 2574f780b..c496ea37c 100644 --- a/src/Elastic.Markdown/Assets/styles.css +++ b/src/Elastic.Markdown/Assets/styles.css @@ -115,7 +115,6 @@ color: var(--color-yellow-80); } - .applicable-info { padding: calc(var(--spacing) * 0.5); padding-left: calc(var(--spacing) * 2); diff --git a/src/Elastic.Markdown/BuildContext.cs b/src/Elastic.Markdown/BuildContext.cs index 3a1825d7f..1d5d26613 100644 --- a/src/Elastic.Markdown/BuildContext.cs +++ b/src/Elastic.Markdown/BuildContext.cs @@ -9,6 +9,7 @@ using Elastic.Markdown.IO; using Elastic.Markdown.IO.Configuration; using Elastic.Markdown.IO.Discovery; +using Elastic.Markdown.IO.State; namespace Elastic.Markdown; diff --git a/src/Elastic.Markdown/Elastic.Markdown.csproj b/src/Elastic.Markdown/Elastic.Markdown.csproj index 9e54a1fb1..3babb2950 100644 --- a/src/Elastic.Markdown/Elastic.Markdown.csproj +++ b/src/Elastic.Markdown/Elastic.Markdown.csproj @@ -62,6 +62,8 @@ + diff --git a/src/Elastic.Markdown/IO/State/ContentSource.cs b/src/Elastic.Markdown/IO/State/ContentSource.cs new file mode 100644 index 000000000..77f9d82c4 --- /dev/null +++ b/src/Elastic.Markdown/IO/State/ContentSource.cs @@ -0,0 +1,21 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.ComponentModel.DataAnnotations; +using System.Text.Json.Serialization; +using NetEscapades.EnumGenerators; + +namespace Elastic.Markdown.IO.State; + +[EnumExtensions] +public enum ContentSource +{ + [Display(Name = "next")] + [JsonStringEnumMemberName("next")] + Next, + + [JsonStringEnumMemberName("current")] + [Display(Name = "current")] + Current +} diff --git a/src/Elastic.Markdown/IO/State/LinkReference.cs b/src/Elastic.Markdown/IO/State/LinkReference.cs index 3bd6f8f85..acf0dd6a1 100644 --- a/src/Elastic.Markdown/IO/State/LinkReference.cs +++ b/src/Elastic.Markdown/IO/State/LinkReference.cs @@ -62,9 +62,15 @@ public record LinkReference public static string SerializeRedirects(Dictionary? redirects) => JsonSerializer.Serialize(redirects, SourceGenerationContext.Default.DictionaryStringLinkRedirect); + public static LinkReference Deserialize(Stream json) => + JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!; + public static LinkReference Deserialize(string json) => JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!; + public static string Serialize(LinkReference reference) => + JsonSerializer.Serialize(reference, SourceGenerationContext.Default.LinkReference); + public static LinkReference Create(DocumentationSet set) { var redirects = set.Configuration.Redirects; diff --git a/src/Elastic.Markdown/Links/CrossLinks/ConfigurationCrossLinkFetcher.cs b/src/Elastic.Markdown/Links/CrossLinks/ConfigurationCrossLinkFetcher.cs index c7f03174f..f62202715 100644 --- a/src/Elastic.Markdown/Links/CrossLinks/ConfigurationCrossLinkFetcher.cs +++ b/src/Elastic.Markdown/Links/CrossLinks/ConfigurationCrossLinkFetcher.cs @@ -21,7 +21,7 @@ public override async Task Fetch(Cancel ctx) _ = declaredRepositories.Add(repository); try { - var linkReference = await Fetch(repository, ctx); + var linkReference = await Fetch(repository, ["main", "master"], ctx); linkReferences.Add(repository, linkReference); var linkIndexReference = await GetLinkIndexEntry(repository, ctx); linkIndexEntries.Add(repository, linkIndexReference); diff --git a/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs b/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs index 1dbe79d9f..6fd7c9acd 100644 --- a/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs +++ b/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs @@ -57,23 +57,34 @@ protected async Task FetchLinkIndex(Cancel ctx) protected async Task GetLinkIndexEntry(string repository, Cancel ctx) { var linkIndex = await FetchLinkIndex(ctx); - if (linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks)) - return repositoryLinks.First().Value; - throw new Exception($"Repository {repository} not found in link index"); + if (!linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks)) + throw new Exception($"Repository {repository} not found in link index"); + return GetNextContentSourceLinkIndexEntry(repositoryLinks, repository); } - protected async Task Fetch(string repository, Cancel ctx) + protected static LinkIndexEntry GetNextContentSourceLinkIndexEntry(IDictionary repositoryLinks, string repository) + { + var linkIndexEntry = + (repositoryLinks.TryGetValue("main", out var link) + ? link + : repositoryLinks.TryGetValue("master", out link) ? link : null) + ?? throw new Exception($"Repository {repository} found in link index, but no main or master branch found"); + return linkIndexEntry; + } + + protected async Task Fetch(string repository, string[] keys, Cancel ctx) { var linkIndex = await FetchLinkIndex(ctx); if (!linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks)) throw new Exception($"Repository {repository} not found in link index"); - if (repositoryLinks.TryGetValue("main", out var linkIndexEntry)) - return await FetchLinkIndexEntry(repository, linkIndexEntry, ctx); - if (repositoryLinks.TryGetValue("master", out linkIndexEntry)) - return await FetchLinkIndexEntry(repository, linkIndexEntry, ctx); - throw new Exception($"Repository {repository} not found in link index"); + foreach (var key in keys) + { + if (repositoryLinks.TryGetValue(key, out var linkIndexEntry)) + return await FetchLinkIndexEntry(repository, linkIndexEntry, ctx); + } + throw new Exception($"Repository found in link index however none of: '{string.Join(", ", keys)}' branches found"); } protected async Task FetchLinkIndexEntry(string repository, LinkIndexEntry linkIndexEntry, Cancel ctx) diff --git a/src/Elastic.Markdown/Links/CrossLinks/CrossLinkResolver.cs b/src/Elastic.Markdown/Links/CrossLinks/CrossLinkResolver.cs index dac9cbf55..dd5b1fa24 100644 --- a/src/Elastic.Markdown/Links/CrossLinks/CrossLinkResolver.cs +++ b/src/Elastic.Markdown/Links/CrossLinks/CrossLinkResolver.cs @@ -14,6 +14,9 @@ public record LinkIndex { [JsonPropertyName("repositories")] public required Dictionary> Repositories { get; init; } + public static LinkIndex Deserialize(Stream json) => + JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkIndex)!; + public static LinkIndex Deserialize(string json) => JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkIndex)!; @@ -34,6 +37,10 @@ public record LinkIndexEntry [JsonPropertyName("etag")] public required string ETag { get; init; } + + // TODO can be made required after all doc_sets have published again + [JsonPropertyName("ref")] + public string GitReference { get; init; } = "unknown"; } public interface ICrossLinkResolver diff --git a/src/Elastic.Markdown/Links/InboundLinks/LinkIndexCrossLinkFetcher.cs b/src/Elastic.Markdown/Links/InboundLinks/LinkIndexCrossLinkFetcher.cs index 6779541f1..d9ecfe50d 100644 --- a/src/Elastic.Markdown/Links/InboundLinks/LinkIndexCrossLinkFetcher.cs +++ b/src/Elastic.Markdown/Links/InboundLinks/LinkIndexCrossLinkFetcher.cs @@ -19,7 +19,8 @@ public override async Task Fetch(Cancel ctx) var linkIndex = await FetchLinkIndex(ctx); foreach (var (repository, value) in linkIndex.Repositories) { - var linkIndexEntry = value.First().Value; + var linkIndexEntry = GetNextContentSourceLinkIndexEntry(value, repository); + linkEntries.Add(repository, linkIndexEntry); var linkReference = await FetchLinkIndexEntry(repository, linkIndexEntry, ctx); linkReferences.Add(repository, linkReference); @@ -34,4 +35,5 @@ public override async Task Fetch(Cancel ctx) FromConfiguration = false }; } + } diff --git a/src/Elastic.Markdown/SourceGenerationContext.cs b/src/Elastic.Markdown/SourceGenerationContext.cs index 3a1b47d5b..850734d8a 100644 --- a/src/Elastic.Markdown/SourceGenerationContext.cs +++ b/src/Elastic.Markdown/SourceGenerationContext.cs @@ -11,7 +11,7 @@ namespace Elastic.Markdown; // This configures the source generation for json (de)serialization. -[JsonSourceGenerationOptions(WriteIndented = true)] +[JsonSourceGenerationOptions(WriteIndented = true, UseStringEnumConverter = true)] [JsonSerializable(typeof(GenerationState))] [JsonSerializable(typeof(LinkReference))] [JsonSerializable(typeof(GitCheckoutInformation))] diff --git a/src/docs-assembler/AssembleContext.cs b/src/docs-assembler/AssembleContext.cs index 5ffe1079c..da6f71679 100644 --- a/src/docs-assembler/AssembleContext.cs +++ b/src/docs-assembler/AssembleContext.cs @@ -7,6 +7,7 @@ using Documentation.Assembler.Configuration; using Elastic.Markdown.Diagnostics; using Elastic.Markdown.IO; +using Elastic.Markdown.IO.State; namespace Documentation.Assembler; diff --git a/src/docs-assembler/AssembleSources.cs b/src/docs-assembler/AssembleSources.cs index a53fc233c..e11a8b553 100644 --- a/src/docs-assembler/AssembleSources.cs +++ b/src/docs-assembler/AssembleSources.cs @@ -59,7 +59,7 @@ private AssembleSources(AssembleContext assembleContext, Checkout[] checkouts) TocTopLevelMappings = GetConfiguredSources(assembleContext); HistoryMappings = GetHistoryMapping(assembleContext); - var crossLinkFetcher = new AssemblerCrossLinkFetcher(NullLoggerFactory.Instance, assembleContext.Configuration); + var crossLinkFetcher = new AssemblerCrossLinkFetcher(NullLoggerFactory.Instance, assembleContext.Configuration, assembleContext.Environment); UriResolver = new PublishEnvironmentUriResolver(TocTopLevelMappings, assembleContext.Environment); var crossLinkResolver = new CrossLinkResolver(crossLinkFetcher, UriResolver); AssembleSets = checkouts diff --git a/src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs b/src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs index a6f5d3831..815a5f442 100644 --- a/src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs +++ b/src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs @@ -10,7 +10,7 @@ namespace Documentation.Assembler.Building; -public class AssemblerCrossLinkFetcher(ILoggerFactory logger, AssemblyConfiguration configuration) : CrossLinkFetcher(logger) +public class AssemblerCrossLinkFetcher(ILoggerFactory logger, AssemblyConfiguration configuration, PublishEnvironment publishEnvironment) : CrossLinkFetcher(logger) { public override async Task Fetch(Cancel ctx) { @@ -27,7 +27,11 @@ public override async Task Fetch(Cancel ctx) if (repository.Skip) continue; - var linkReference = await Fetch(repositoryName, ctx); + var branch = publishEnvironment.ContentSource == ContentSource.Current + ? repository.GitReferenceCurrent + : repository.GitReferenceNext; + + var linkReference = await Fetch(repositoryName, [branch], ctx); linkReferences.Add(repositoryName, linkReference); var linkIndexReference = await GetLinkIndexEntry(repositoryName, ctx); linkIndexEntries.Add(repositoryName, linkIndexReference); diff --git a/src/docs-assembler/Configuration/AssemblyConfiguration.cs b/src/docs-assembler/Configuration/AssemblyConfiguration.cs index 36cd68281..c8ad25452 100644 --- a/src/docs-assembler/Configuration/AssemblyConfiguration.cs +++ b/src/docs-assembler/Configuration/AssemblyConfiguration.cs @@ -2,8 +2,7 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information -using System.ComponentModel.DataAnnotations; -using NetEscapades.EnumGenerators; +using Elastic.Markdown.IO.State; using YamlDotNet.Serialization; namespace Documentation.Assembler.Configuration; @@ -101,16 +100,6 @@ public record PublishEnvironment public GoogleTagManager GoogleTagManager { get; set; } = new(); } -[EnumExtensions] -public enum ContentSource -{ - [Display(Name = "next")] - Next, - - [Display(Name = "current")] - Current -} - public record GoogleTagManager { [YamlMember(Alias = "enabled")] diff --git a/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs b/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs index 84de2c886..1220ef3d4 100644 --- a/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs +++ b/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs @@ -8,6 +8,7 @@ using System.IO.Abstractions; using Documentation.Assembler.Configuration; using Elastic.Markdown.IO; +using Elastic.Markdown.IO.State; using Microsoft.Extensions.Logging; using ProcNet; using ProcNet.Std; diff --git a/src/docs-assembler/YamlStaticContext.cs b/src/docs-assembler/YamlStaticContext.cs index e2963ece4..118ec943b 100644 --- a/src/docs-assembler/YamlStaticContext.cs +++ b/src/docs-assembler/YamlStaticContext.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using Documentation.Assembler.Configuration; +using Elastic.Markdown.IO.State; using YamlDotNet.Serialization; namespace Documentation.Assembler; diff --git a/src/docs-assembler/docs-assembler.csproj b/src/docs-assembler/docs-assembler.csproj index fbf15664f..cdd011132 100644 --- a/src/docs-assembler/docs-assembler.csproj +++ b/src/docs-assembler/docs-assembler.csproj @@ -23,8 +23,6 @@ - diff --git a/src/infra/docs-lambda-index-publisher/Program.cs b/src/infra/docs-lambda-index-publisher/Program.cs index 56e3518da..16c8814bf 100644 --- a/src/infra/docs-lambda-index-publisher/Program.cs +++ b/src/infra/docs-lambda-index-publisher/Program.cs @@ -8,6 +8,7 @@ using Amazon.Lambda.RuntimeSupport; using Amazon.S3; using Amazon.S3.Model; +using Elastic.Markdown.IO.State; using Elastic.Markdown.Links.CrossLinks; await LambdaBootstrapBuilder.Create(Handler) @@ -44,6 +45,8 @@ static async Task Handler(ILambdaContext context) if (tokens.Length < 3) continue; + var gitReference = await ReadLinkReferenceSha(client, obj); + var repository = tokens[1]; var branch = tokens[2]; @@ -52,7 +55,8 @@ static async Task Handler(ILambdaContext context) Repository = repository, Branch = branch, ETag = obj.ETag.Trim('"'), - Path = obj.Key + Path = obj.Key, + GitReference = gitReference }; if (linkIndex.Repositories.TryGetValue(repository, out var existingEntry)) existingEntry[branch] = entry; @@ -83,3 +87,21 @@ static async Task Handler(ILambdaContext context) await client.UploadObjectFromStreamAsync(bucketName, "link-index.json", stream, new Dictionary(), CancellationToken.None); return $"Finished in {sw}"; } + +static async Task ReadLinkReferenceSha(IAmazonS3 client, S3Object obj) +{ + try + { + var contents = await client.GetObjectAsync(obj.Key, obj.Key, CancellationToken.None); + await using var s = contents.ResponseStream; + var linkReference = LinkReference.Deserialize(s); + return linkReference.Origin.Ref; + } + catch (Exception e) + { + Console.WriteLine(e); + // it's important we don't fail here we need to fallback gracefully from this so we can fix the root cause + // of why a repository is not reporting its git reference properly + return "unknown"; + } +} diff --git a/tests/Elastic.Markdown.Tests/DocSet/LinkReferenceTests.cs b/tests/Elastic.Markdown.Tests/DocSet/LinkReferenceTests.cs index 5760a5c54..d11ff457f 100644 --- a/tests/Elastic.Markdown.Tests/DocSet/LinkReferenceTests.cs +++ b/tests/Elastic.Markdown.Tests/DocSet/LinkReferenceTests.cs @@ -26,6 +26,7 @@ public void EmitsLinks() => [Fact] public void ShouldNotIncludeSnippets() => Reference.Links.Should().NotContain(l => l.Key.Contains("_snippets/")); + } public class GitCheckoutInformationTests(ITestOutputHelper output) : NavigationTestsBase(output) @@ -47,3 +48,64 @@ public void Create() git.Remote.Should().NotContain(".git"); } } + +public class LinkReferenceSerializationTests +{ + [Fact] + public void SerializesCurrent() + { + var linkReference = new LinkReference + { + Origin = new GitCheckoutInformation + { + Branch = "branch", + Remote = "remote", + Ref = "ref" + }, + UrlPathPrefix = "", + Links = [], + CrossLinks = [], + }; + var json = LinkReference.Serialize(linkReference); + // language=json + json.Should().Be( + """ + { + "origin": { + "branch": "branch", + "remote": "remote", + "ref": "ref", + "name": "unavailable" + }, + "url_path_prefix": "", + "links": {}, + "cross_links": [], + "redirects": null + } + """); + } + + [Fact] + public void Deserializes() + { + // language=json + var json = + """ + { + "origin": { + "branch": "branch", + "remote": "remote", + "ref": "ref", + "name": "unavailable" + }, + "url_path_prefix": "", + "links": {}, + "cross_links": [], + "redirects": null + } + """; + var linkReference = LinkReference.Deserialize(json); + linkReference.Origin.Ref.Should().Be("ref"); + } + +} diff --git a/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs b/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs index 97b0be678..6d739f2a6 100644 --- a/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs +++ b/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs @@ -22,6 +22,7 @@ public Task FetchLinks(Cancel ctx) // language=json var json = """ { + "content_source": "current", "origin": { "branch": "main", "remote": " https://github.com/elastic/docs-content", @@ -53,7 +54,8 @@ public Task FetchLinks(Cancel ctx) Repository = e.Key, Path = $"elastic/asciidocalypse/{e.Key}/links.json", Branch = "main", - ETag = Guid.NewGuid().ToString() + ETag = Guid.NewGuid().ToString(), + GitReference = Guid.NewGuid().ToString() }); _crossLinks = new FetchedCrossLinks { diff --git a/tests/authoring/Framework/TestCrossLinkResolver.fs b/tests/authoring/Framework/TestCrossLinkResolver.fs index 5ee073063..cc97b3027 100644 --- a/tests/authoring/Framework/TestCrossLinkResolver.fs +++ b/tests/authoring/Framework/TestCrossLinkResolver.fs @@ -73,7 +73,8 @@ type TestCrossLinkResolver (config: ConfigurationFile) = Repository = e.Key, Path = $"elastic/asciidocalypse/{e.Key}/links.json", Branch = "main", - ETag = Guid.NewGuid().ToString() + ETag = Guid.NewGuid().ToString(), + GitReference = Guid.NewGuid().ToString() )) let crossLinks = @@ -91,7 +92,8 @@ type TestCrossLinkResolver (config: ConfigurationFile) = Repository = e.Key, Path = $"elastic/asciidocalypse/{e.Key}/links.json", Branch = "main", - ETag = Guid.NewGuid().ToString() + ETag = Guid.NewGuid().ToString(), + GitReference = Guid.NewGuid().ToString() )); let crossLinks = diff --git a/tests/docs-assembler.Tests/src/docs-assembler.Tests/AssemblerConfigurationTests.cs b/tests/docs-assembler.Tests/src/docs-assembler.Tests/AssemblerConfigurationTests.cs index b88b2bd66..d28d91b74 100644 --- a/tests/docs-assembler.Tests/src/docs-assembler.Tests/AssemblerConfigurationTests.cs +++ b/tests/docs-assembler.Tests/src/docs-assembler.Tests/AssemblerConfigurationTests.cs @@ -6,6 +6,7 @@ using Documentation.Assembler.Configuration; using Elastic.Markdown.Diagnostics; using Elastic.Markdown.IO; +using Elastic.Markdown.IO.State; using FluentAssertions; namespace Documentation.Assembler.Tests;