diff --git a/src/Elastic.Markdown/Elastic.Markdown.csproj b/src/Elastic.Markdown/Elastic.Markdown.csproj index 6ef4b4150..e71ea3b59 100644 --- a/src/Elastic.Markdown/Elastic.Markdown.csproj +++ b/src/Elastic.Markdown/Elastic.Markdown.csproj @@ -59,7 +59,7 @@ - + diff --git a/src/Elastic.Markdown/Myst/YamlSerialization.cs b/src/Elastic.Markdown/Myst/YamlSerialization.cs index 1ae8e1ca6..9cdf134db 100644 --- a/src/Elastic.Markdown/Myst/YamlSerialization.cs +++ b/src/Elastic.Markdown/Myst/YamlSerialization.cs @@ -39,5 +39,4 @@ public static T Deserialize(string yaml) [YamlSerializable(typeof(Setting))] [YamlSerializable(typeof(AllowedValue))] [YamlSerializable(typeof(SettingMutability))] - public partial class DocsBuilderYamlStaticContext; diff --git a/src/docs-assembler/AssembleContext.cs b/src/docs-assembler/AssembleContext.cs index ae1d5720d..ab3aff075 100644 --- a/src/docs-assembler/AssembleContext.cs +++ b/src/docs-assembler/AssembleContext.cs @@ -67,13 +67,15 @@ public AssembleContext( ExtractAssemblerConfiguration(historyMappingPath, "historymapping.yml"); HistoryMappingPath = ReadFileSystem.FileInfo.New(historyMappingPath); - CheckoutDirectory = ReadFileSystem.DirectoryInfo.New(checkoutDirectory ?? ".artifacts/checkouts"); - OutputDirectory = ReadFileSystem.DirectoryInfo.New(output ?? ".artifacts/assembly"); - - if (!Configuration.Environments.TryGetValue(environment, out var env)) throw new Exception($"Could not find environment {environment}"); Environment = env; + + var contentSource = Environment.ContentSource.ToStringFast(); + CheckoutDirectory = ReadFileSystem.DirectoryInfo.New(checkoutDirectory ?? Path.Combine(".artifacts", "checkouts", contentSource)); + OutputDirectory = ReadFileSystem.DirectoryInfo.New(output ?? Path.Combine(".artifacts", "assembly")); + + } private void ExtractAssemblerConfiguration(string configPath, string file) diff --git a/src/docs-assembler/Cli/RepositoryCommands.cs b/src/docs-assembler/Cli/RepositoryCommands.cs index 64c6514de..3c283d59f 100644 --- a/src/docs-assembler/Cli/RepositoryCommands.cs +++ b/src/docs-assembler/Cli/RepositoryCommands.cs @@ -29,15 +29,22 @@ private void AssignOutputLogger() // libgit2 is magnitudes slower to clone repositories https://github.com/libgit2/libgit2/issues/4674 /// Clones all repositories /// Treat warnings as errors and fail the build on warnings + /// The environment to build /// [Command("clone-all")] - public async Task CloneAll(bool? strict = null, Cancel ctx = default) + public async Task CloneAll( + bool? strict = null, + string? environment = null, + Cancel ctx = default + ) { AssignOutputLogger(); + var githubEnvironmentInput = githubActionsService.GetInput("environment"); + environment ??= !string.IsNullOrEmpty(githubEnvironmentInput) ? githubEnvironmentInput : "dev"; await using var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService); - var assembleContext = new AssembleContext("dev", collector, new FileSystem(), new FileSystem(), null, null); + var assembleContext = new AssembleContext(environment, collector, new FileSystem(), new FileSystem(), null, null); var cloner = new AssemblerRepositorySourcer(logger, assembleContext); _ = await cloner.AcquireAllLatest(ctx); @@ -50,7 +57,7 @@ public async Task CloneAll(bool? strict = null, Cancel ctx = default) /// Force a full rebuild of the destination folder /// Treat warnings as errors and fail the build on warnings /// Allow indexing and following of html files - /// The environment to resolve links to + /// The environment to build /// [Command("build-all")] public async Task BuildAll( diff --git a/src/docs-assembler/Configuration/AssemblyConfiguration.cs b/src/docs-assembler/Configuration/AssemblyConfiguration.cs index 383f863e4..36cd68281 100644 --- a/src/docs-assembler/Configuration/AssemblyConfiguration.cs +++ b/src/docs-assembler/Configuration/AssemblyConfiguration.cs @@ -2,6 +2,8 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.ComponentModel.DataAnnotations; +using NetEscapades.EnumGenerators; using YamlDotNet.Serialization; namespace Documentation.Assembler.Configuration; @@ -24,6 +26,7 @@ public static AssemblyConfiguration Deserialize(string yaml) var repository = RepositoryDefaults(r, name); config.ReferenceRepositories[name] = repository; } + foreach (var (name, env) in config.Environments) env.Name = name; config.Narrative = RepositoryDefaults(config.Narrative, NarrativeRepository.RepositoryName); @@ -44,8 +47,10 @@ private static TRepository RepositoryDefaults(TRepository r, string var repository = r ?? new TRepository(); // ReSharper restore NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract repository.Name = name; - if (string.IsNullOrEmpty(repository.CurrentBranch)) - repository.CurrentBranch = "main"; + if (string.IsNullOrEmpty(repository.GitReferenceCurrent)) + repository.GitReferenceCurrent = "main"; + if (string.IsNullOrEmpty(repository.GitReferenceNext)) + repository.GitReferenceNext = "main"; if (string.IsNullOrEmpty(repository.Origin)) { if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GITHUB_ACTIONS"))) @@ -70,6 +75,9 @@ private static TRepository RepositoryDefaults(TRepository r, string [YamlMember(Alias = "environments")] public Dictionary Environments { get; set; } = []; + + [YamlMember(Alias = "named_git_references")] + public Dictionary NamedGitReferences { get; set; } = []; } public record PublishEnvironment @@ -86,16 +94,30 @@ public record PublishEnvironment [YamlMember(Alias = "allow_indexing")] public bool AllowIndexing { get; set; } + [YamlMember(Alias = "content_source")] + public ContentSource ContentSource { get; set; } + [YamlMember(Alias = "google_tag_manager")] public GoogleTagManager GoogleTagManager { get; set; } = new(); } +[EnumExtensions] +public enum ContentSource +{ + [Display(Name = "next")] + Next, + + [Display(Name = "current")] + Current +} + public record GoogleTagManager { [YamlMember(Alias = "enabled")] public bool Enabled { get; set; } private string _id = string.Empty; + [YamlMember(Alias = "id")] public string Id { @@ -107,6 +129,7 @@ public string Id _id = value; } } + [YamlMember(Alias = "auth")] public string? Auth { get; set; } diff --git a/src/docs-assembler/Configuration/Repository.cs b/src/docs-assembler/Configuration/Repository.cs index 46ad8462f..1d783e0af 100644 --- a/src/docs-assembler/Configuration/Repository.cs +++ b/src/docs-assembler/Configuration/Repository.cs @@ -21,7 +21,10 @@ public record Repository public string Origin { get; set; } = string.Empty; [YamlMember(Alias = "current")] - public string CurrentBranch { get; set; } = "main"; + public string GitReferenceCurrent { get; set; } = "main"; + + [YamlMember(Alias = "next")] + public string GitReferenceNext { get; set; } = "main"; [YamlMember(Alias = "checkout_strategy")] public string CheckoutStrategy { get; set; } = "partial"; diff --git a/src/docs-assembler/Navigation/AssemblerDocumentationSet.cs b/src/docs-assembler/Navigation/AssemblerDocumentationSet.cs index a5ce6925f..38b5c3c25 100644 --- a/src/docs-assembler/Navigation/AssemblerDocumentationSet.cs +++ b/src/docs-assembler/Navigation/AssemblerDocumentationSet.cs @@ -44,7 +44,7 @@ public AssemblerDocumentationSet( RepositoryName = checkout.Repository.Name, Ref = checkout.HeadReference, Remote = $"elastic/${checkout.Repository.Name}", - Branch = checkout.Repository.CurrentBranch + Branch = checkout.Repository.GitReferenceCurrent }; var buildContext = new BuildContext( diff --git a/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs b/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs index 8468604bd..35c0216c9 100644 --- a/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs +++ b/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs @@ -4,6 +4,7 @@ using System.Collections.Concurrent; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO.Abstractions; using Documentation.Assembler.Configuration; using Elastic.Markdown.IO; @@ -18,6 +19,7 @@ public class AssemblerRepositorySourcer(ILoggerFactory logger, AssembleContext c private readonly ILogger _logger = logger.CreateLogger(); private AssemblyConfiguration Configuration => context.Configuration; + private PublishEnvironment PublishEnvironment => context.Environment; public IReadOnlyCollection GetAll() { @@ -27,20 +29,28 @@ public IReadOnlyCollection GetAll() foreach (var repo in repositories) { var checkoutFolder = fs.DirectoryInfo.New(Path.Combine(context.CheckoutDirectory.FullName, repo.Name)); - //var head = Capture(checkoutFolder, "git", "rev-parse", "HEAD"); var checkout = new Checkout { Repository = repo, Directory = checkoutFolder, + //TODO read from links.json and ensure we check out exactly that git reference + //+ validate that git reference belongs to the appropriate branch HeadReference = Guid.NewGuid().ToString("N") }; checkouts.Add(checkout); } + return checkouts; } public async Task> AcquireAllLatest(Cancel ctx = default) { + _logger.LogInformation( + "Cloning all repositories for environment {EnvironmentName} using '{ContentSourceStrategy}' content sourcing strategy", + PublishEnvironment.Name, + PublishEnvironment.ContentSource.ToStringFast() + ); + var dict = new ConcurrentDictionary(); var checkouts = new ConcurrentBag(); @@ -65,7 +75,7 @@ await Task.Run(() => }).ConfigureAwait(false); foreach (var kv in dict.OrderBy(kv => kv.Value.Elapsed)) - _logger.LogInformation("-> {Repository}\ttook: {Elapsed}", kv.Key, kv.Value.Elapsed); + _logger.LogInformation("-> took: {Elapsed}\t{RepositoryBranch}", kv.Key, kv.Value.Elapsed); return checkouts.ToList().AsReadOnly(); } @@ -76,39 +86,20 @@ private Checkout CloneOrUpdateRepository(Repository repository, string name, Con var checkoutFolder = fs.DirectoryInfo.New(Path.Combine(context.CheckoutDirectory.FullName, name)); var relativePath = Path.GetRelativePath(Paths.WorkingDirectoryRoot.FullName, checkoutFolder.FullName); var sw = Stopwatch.StartNew(); - _ = dict.AddOrUpdate(name, sw, (_, _) => sw); - var head = string.Empty; + var branch = PublishEnvironment.ContentSource == ContentSource.Next + ? repository.GitReferenceNext + : repository.GitReferenceCurrent; + + _ = dict.AddOrUpdate($"{name} ({branch})", sw, (_, _) => sw); + + string? head; if (checkoutFolder.Exists) { - _logger.LogInformation("Pull: {Name}\t{Repository}\t{RelativePath}", name, repository, relativePath); - // --allow-unrelated-histories due to shallow clones not finding a common ancestor - ExecIn(checkoutFolder, "git", "pull", "--depth", "1", "--allow-unrelated-histories", "--no-ff"); - //head = Capture(checkoutFolder, "git", "rev-parse", "HEAD"); - head = Guid.NewGuid().ToString("N"); + if (!TryUpdateSource(name, branch, relativePath, checkoutFolder, out head)) + head = CheckoutFromScratch(repository, name, branch, relativePath, checkoutFolder); } else - { - _logger.LogInformation("Checkout: {Name}\t{Repository}\t{RelativePath}", name, repository, relativePath); - if (repository.CheckoutStrategy == "full") - { - Exec("git", "clone", repository.Origin, checkoutFolder.FullName, - "--depth", "1", "--single-branch", - "--branch", repository.CurrentBranch - ); - } - else if (repository.CheckoutStrategy == "partial") - { - Exec( - "git", "clone", "--filter=blob:none", "--no-checkout", repository.Origin, checkoutFolder.FullName - ); - - ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "--cone"); - ExecIn(checkoutFolder, "git", "checkout", repository.CurrentBranch); - ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "docs"); - //head = Capture(checkoutFolder, "git", "rev-parse", "HEAD"); - head = Guid.NewGuid().ToString("N"); - } - } + head = CheckoutFromScratch(repository, name, branch, relativePath, checkoutFolder); sw.Stop(); @@ -120,6 +111,55 @@ private Checkout CloneOrUpdateRepository(Repository repository, string name, Con }; } + private bool TryUpdateSource(string name, string branch, string relativePath, IDirectoryInfo checkoutFolder, [NotNullWhen(true)] out string? head) + { + head = null; + try + { + _logger.LogInformation("Pull: {Name}\t{Branch}\t{RelativePath}", name, branch, relativePath); + // --allow-unrelated-histories due to shallow clones not finding a common ancestor + ExecIn(checkoutFolder, "git", "pull", "--depth", "1", "--allow-unrelated-histories", "--no-ff"); + head = Capture(checkoutFolder, "git", "rev-parse", "HEAD"); + return true; + } + catch (Exception e) + { + _logger.LogError(e, "Failed to update {Name} from {RelativePath}, falling back to recreating from scratch", name, relativePath); + if (checkoutFolder.Exists) + { + checkoutFolder.Delete(true); + checkoutFolder.Refresh(); + } + } + + return false; + } + + private string CheckoutFromScratch(Repository repository, string name, string branch, string relativePath, + IDirectoryInfo checkoutFolder) + { + _logger.LogInformation("Checkout: {Name}\t{Branch}\t{RelativePath}", name, branch, relativePath); + if (repository.CheckoutStrategy == "full") + { + Exec("git", "clone", repository.Origin, checkoutFolder.FullName, + "--depth", "1", "--single-branch", + "--branch", branch + ); + } + else if (repository.CheckoutStrategy == "partial") + { + Exec( + "git", "clone", "--filter=blob:none", "--no-checkout", repository.Origin, checkoutFolder.FullName + ); + + ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "--cone"); + ExecIn(checkoutFolder, "git", "checkout", branch); + ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "docs"); + } + + return Capture(checkoutFolder, "git", "rev-parse", "HEAD"); + } + private void Exec(string binary, params string[] args) => ExecIn(null, binary, args); private void ExecIn(IDirectoryInfo? workingDirectory, string binary, params string[] args) @@ -136,19 +176,36 @@ private void ExecIn(IDirectoryInfo? workingDirectory, string binary, params stri // ReSharper disable once UnusedMember.Local private string Capture(IDirectoryInfo? workingDirectory, string binary, params string[] args) { - var arguments = new StartArguments(binary, args) + // Try 10 times to capture the output of the command, if it fails we'll throw an exception on the last try + for (var i = 0; i < 9; i++) { - WorkingDirectory = workingDirectory?.FullName, - //WaitForStreamReadersTimeout = TimeSpan.FromSeconds(3), - Timeout = TimeSpan.FromSeconds(3), - WaitForExit = TimeSpan.FromSeconds(3), - ConsoleOutWriter = NoopConsoleWriter.Instance - }; - var result = Proc.Start(arguments); - if (result.ExitCode != 0) - context.Collector.EmitError("", $"Exit code: {result.ExitCode} while executing {binary} {string.Join(" ", args)} in {workingDirectory}"); - var line = result.ConsoleOut.FirstOrDefault()?.Line ?? throw new Exception($"No output captured for {binary}: {workingDirectory}"); - return line; + try + { + return CaptureOutput(); + } + catch + { + // ignored + } + } + return CaptureOutput(); + + string CaptureOutput() + { + var arguments = new StartArguments(binary, args) + { + WorkingDirectory = workingDirectory?.FullName, + //WaitForStreamReadersTimeout = TimeSpan.FromSeconds(3), + Timeout = TimeSpan.FromSeconds(3), + WaitForExit = TimeSpan.FromSeconds(3), + ConsoleOutWriter = NoopConsoleWriter.Instance + }; + var result = Proc.Start(arguments); + if (result.ExitCode != 0) + context.Collector.EmitError("", $"Exit code: {result.ExitCode} while executing {binary} {string.Join(" ", args)} in {workingDirectory}"); + var line = result.ConsoleOut.FirstOrDefault()?.Line ?? throw new Exception($"No output captured for {binary}: {workingDirectory}"); + return line; + } } } diff --git a/src/docs-assembler/YamlStaticContext.cs b/src/docs-assembler/YamlStaticContext.cs index 7fff8253d..e2963ece4 100644 --- a/src/docs-assembler/YamlStaticContext.cs +++ b/src/docs-assembler/YamlStaticContext.cs @@ -13,4 +13,5 @@ namespace Documentation.Assembler; [YamlSerializable(typeof(NarrativeRepository))] [YamlSerializable(typeof(PublishEnvironment))] [YamlSerializable(typeof(GoogleTagManager))] +[YamlSerializable(typeof(ContentSource))] public partial class YamlStaticContext; diff --git a/src/docs-assembler/assembler.yml b/src/docs-assembler/assembler.yml index 1b01e8076..5f2ca8cc4 100644 --- a/src/docs-assembler/assembler.yml +++ b/src/docs-assembler/assembler.yml @@ -2,6 +2,7 @@ environments: prod: uri: https://www.elastic.co path_prefix: docs + content_source: current # current | next allow_indexing: false google_tag_manager: enabled: true @@ -9,6 +10,7 @@ environments: staging: uri: https://staging-website.elastic.co path_prefix: docs + content_source: next google_tag_manager: enabled: true id: GTM-KNJMG2M @@ -17,9 +19,31 @@ environments: cookies_win: x dev: uri: http://localhost:4000 + content_source: next path_prefix: docs + +named_git_references: + stack: &stack 9.0 + cloud-hosted: ms-120 + +### +# 'narrative' shares the same keys as keys in 'references' () +# 'narrative' defines the docs-content repository +### narrative: checkout_strategy: full + +### +# 'references' defines a map of `elastic/ * +# repository_config: +# skip: bool +# checkout_strategy: full | partial +# # 'full' git clone --depth-1 --single-branch +# # 'partial' --cone sparse-checkout of only the 'docs' folder with --filter=blob:none +# current: +# next: +### + references: apm-server: apm-agent-android: @@ -38,12 +62,15 @@ references: cloud-on-k8s: cloud: current: master + next: master curator: current: master + next: master detection-rules: checkout_strategy: full ecctl: current: master + next: master ecs-dotnet: ecs-logging-go-logrus: ecs-logging-go-zap: @@ -68,6 +95,7 @@ references: # skip: true elasticsearch-ruby: elasticsearch: + current: *stack go-elasticsearch: integrations: kibana: diff --git a/src/docs-assembler/docs-assembler.csproj b/src/docs-assembler/docs-assembler.csproj index 10af856e8..7a1ac1307 100644 --- a/src/docs-assembler/docs-assembler.csproj +++ b/src/docs-assembler/docs-assembler.csproj @@ -22,6 +22,11 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/tests/docs-assembler.Tests/src/docs-assembler.Tests/AssemblerConfigurationTests.cs b/tests/docs-assembler.Tests/src/docs-assembler.Tests/AssemblerConfigurationTests.cs new file mode 100644 index 000000000..b88b2bd66 --- /dev/null +++ b/tests/docs-assembler.Tests/src/docs-assembler.Tests/AssemblerConfigurationTests.cs @@ -0,0 +1,64 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using Documentation.Assembler.Configuration; +using Elastic.Markdown.Diagnostics; +using Elastic.Markdown.IO; +using FluentAssertions; + +namespace Documentation.Assembler.Tests; + +public class AssemblerConfigurationTests +{ + private DiagnosticsCollector Collector { get; } + private AssembleContext Context { get; } + private FileSystem FileSystem { get; } + private IDirectoryInfo CheckoutDirectory { get; set; } + public AssemblerConfigurationTests() + { + FileSystem = new FileSystem(); + CheckoutDirectory = FileSystem.DirectoryInfo.New( + FileSystem.Path.Combine(Paths.GetSolutionDirectory()!.FullName, ".artifacts", "checkouts") + ); + Collector = new DiagnosticsCollector([]); + Context = new AssembleContext("dev", Collector, FileSystem, FileSystem, CheckoutDirectory.FullName, null); + } + + [Fact] + public void ReadsContentSource() + { + var environments = Context.Configuration.Environments; + environments.Should().NotBeEmpty() + .And.ContainKey("prod"); + + var prod = environments["prod"]; + prod.ContentSource.Should().Be(ContentSource.Current); + + var staging = environments["staging"]; + staging.ContentSource.Should().Be(ContentSource.Next); + } + + [Fact] + public void ReadsVersions() + { + var config = Context.Configuration; + config.NamedGitReferences.Should().NotBeEmpty() + .And.ContainKey("stack"); + + config.NamedGitReferences["stack"].Should().NotBeNullOrEmpty(); + + var agent = config.ReferenceRepositories["elasticsearch"]; + agent.GitReferenceCurrent.Should().NotBeNullOrEmpty() + .And.Be(config.NamedGitReferences["stack"]); + + // test defaults + var apmServer = config.ReferenceRepositories["apm-server"]; + apmServer.GitReferenceNext.Should().NotBeNullOrEmpty() + .And.Be("main"); + apmServer.GitReferenceCurrent.Should().NotBeNullOrEmpty() + .And.Be("main"); + + } +}